merge_from_chromium.py revision bda7cff5dd92725ba000fe95d9434f81a7af486e
1#!/usr/bin/python 2# 3# Copyright (C) 2012 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Merge Chromium into the Android tree.""" 18 19import contextlib 20import logging 21import optparse 22import os 23import re 24import sys 25import urllib2 26 27import merge_common 28 29 30# We need to import this *after* merging from upstream to get the latest 31# version. Set it to none here to catch uses before it's imported. 32webview_licenses = None 33 34 35AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' 36 37 38def _ReadGitFile(git_url, git_branch, sha1, path): 39 """Reads a file from a remote git project at a specific revision. 40 41 Args: 42 git_url: The URL of the git server. 43 git_branch: The branch to fetch. 44 sha1: The SHA1 at which to read. 45 path: The relative path of the file to read. 46 Returns: 47 The contents of the specified file. 48 """ 49 # We fetch the branch to a temporary head so that we don't download the same 50 # commits multiple times. 51 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, 52 git_branch + ':cached_upstream']) 53 return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)]) 54 55 56def _ParseDEPS(git_url, git_branch, sha1): 57 """Parses the .DEPS.git file from Chromium and returns its contents. 58 59 Args: 60 git_url: The URL of the git server. 61 git_branch: The branch to read. 62 sha1: The SHA1 at which to read. 63 Returns: 64 A dictionary of the contents of .DEPS.git at the specified revision 65 """ 66 67 class FromImpl(object): 68 """Used to implement the From syntax.""" 69 70 def __init__(self, module_name): 71 self.module_name = module_name 72 73 def __str__(self): 74 return 'From("%s")' % self.module_name 75 76 class _VarImpl(object): 77 def __init__(self, custom_vars, local_scope): 78 self._custom_vars = custom_vars 79 self._local_scope = local_scope 80 81 def Lookup(self, var_name): 82 """Implements the Var syntax.""" 83 if var_name in self._custom_vars: 84 return self._custom_vars[var_name] 85 elif var_name in self._local_scope.get('vars', {}): 86 return self._local_scope['vars'][var_name] 87 raise Exception('Var is not defined: %s' % var_name) 88 89 tmp_locals = {} 90 var = _VarImpl({}, tmp_locals) 91 tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} 92 deps_content = _ReadGitFile(git_url, git_branch, sha1, '.DEPS.git') 93 exec(deps_content) in tmp_globals, tmp_locals 94 return tmp_locals 95 96 97def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars): 98 """Gets the git URL and SHA1 for each project based on .DEPS.git. 99 100 Args: 101 third_party_projects: The list of projects to consider. 102 deps_vars: The dictionary of dependencies from .DEPS.git. 103 Returns: 104 A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' 105 Raises: 106 MergeError: if a project to be merged is not found in .DEPS.git. 107 """ 108 deps_fallback_order = [ 109 deps_vars['deps'], 110 deps_vars['deps_os']['unix'], 111 deps_vars['deps_os']['android'], 112 ] 113 result = {} 114 for path in third_party_projects: 115 for deps in deps_fallback_order: 116 url_plus_sha1 = deps.get(os.path.join('src', path)) 117 if url_plus_sha1: 118 break 119 else: 120 raise merge_common.MergeError( 121 'Could not find .DEPS.git entry for project %s. This probably ' 122 'means that the project list in merge_from_chromium.py needs to be ' 123 'updated.' % path) 124 match = re.match('(.*?)@(.*)', url_plus_sha1) 125 url = match.group(1) 126 sha1 = match.group(2) 127 logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path) 128 result[path] = {'url': url, 'sha1': sha1} 129 return result 130 131 132def _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended): 133 """Merges each required Chromium project into the Android repository. 134 135 .DEPS.git is consulted to determine which revision each project must be merged 136 at. Only a whitelist of required projects are merged. 137 138 Args: 139 git_url: The URL of the Chromium repository to merge from. 140 git_branch: The branch in the Chromium repository to merge from. 141 svn_revision: The SVN revision in the Chromium repository to merge from. 142 root_sha1: The git hash corresponding to svn_revision. 143 unattended: Run in unattended mode. 144 Raises: 145 MergeError: If incompatibly licensed code is left after pruning. 146 """ 147 # The logic for this step lives here, in the Android tree, as it makes no 148 # sense for a Chromium tree to know about this merge. 149 150 if unattended: 151 branch_create_flag = '-B' 152 else: 153 branch_create_flag = '-b' 154 155 logging.debug('Parsing DEPS ...') 156 deps_vars = _ParseDEPS(git_url, git_branch, root_sha1) 157 158 merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, 159 deps_vars) 160 161 for path in merge_info: 162 url = merge_info[path]['url'] 163 sha1 = merge_info[path]['sha1'] 164 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 165 merge_common.GetCommandStdout(['git', 'checkout', 166 branch_create_flag, 'merge-from-chromium', 167 '-t', 'goog/master-chromium'], cwd=dest_dir) 168 logging.debug('Fetching project %s at %s ...', path, sha1) 169 merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir) 170 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], 171 cwd=dest_dir): 172 logging.debug('Merging project %s at %s ...', path, sha1) 173 # Merge conflicts make git merge return 1, so ignore errors 174 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], 175 cwd=dest_dir, ignore_errors=True) 176 merge_common.CheckNoConflictsAndCommitMerge( 177 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), 178 cwd=dest_dir, unattended=unattended) 179 else: 180 logging.debug('No new commits to merge in project %s', path) 181 182 # Handle root repository separately. 183 merge_common.GetCommandStdout(['git', 'checkout', 184 branch_create_flag, 'merge-from-chromium', 185 '-t', 'goog/master-chromium']) 186 logging.debug('Fetching Chromium at %s ...', root_sha1) 187 merge_common.GetCommandStdout(['git', 'fetch', git_url, git_branch]) 188 logging.debug('Merging Chromium at %s ...', root_sha1) 189 # Merge conflicts make git merge return 1, so ignore errors 190 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1], 191 ignore_errors=True) 192 merge_common.CheckNoConflictsAndCommitMerge( 193 'Merge Chromium from %s branch %s at r%s (%s)\n\n%s' 194 % (git_url, git_branch, svn_revision, root_sha1, AUTOGEN_MESSAGE), 195 unattended=unattended) 196 197 logging.debug('Getting directories to exclude ...') 198 199 # We import this now that we have merged the latest version. 200 # It imports to a global in order that it can be used to generate NOTICE 201 # later. We also disable writing bytecode to keep the source tree clean. 202 sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', 203 'tools')) 204 sys.dont_write_bytecode = True 205 global webview_licenses 206 import webview_licenses 207 import known_issues 208 209 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): 210 logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in 211 exclude_list)) 212 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 213 merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + 214 exclude_list, cwd=dest_dir) 215 if _ModifiedFilesInIndex(dest_dir): 216 merge_common.GetCommandStdout(['git', 'commit', '-m', 217 'Exclude incompatible directories'], 218 cwd=dest_dir) 219 220 directories_left_over = webview_licenses.GetIncompatibleDirectories() 221 if directories_left_over: 222 raise merge_common.MergeError('Incompatibly licensed directories remain: ' + 223 '\n'.join(directories_left_over)) 224 225 226def _GenerateMakefiles(svn_revision): 227 """Run gyp to generate the Android build system makefiles. 228 229 Args: 230 svn_revision: The SVN revision to mention in generated commit messages. 231 """ 232 # TODO(torne): The .tmp files are generated by 233 # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source 234 # tree. We should avoid this, or at least use a more specific name to avoid 235 # accidentally removing or adding other files. 236 for path in merge_common.ALL_PROJECTS: 237 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 238 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', 239 'GypAndroid.*.mk', '*.target.*.mk', 240 '*.host.*.mk', '*.tmp'], cwd=dest_dir) 241 242 merge_common.GetCommandStdout(['android_webview/tools/gyp_webview']) 243 244 for path in merge_common.ALL_PROJECTS: 245 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 246 # git add doesn't have an --ignore-unmatch so we have to do this instead: 247 merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], 248 ignore_errors=True, cwd=dest_dir) 249 merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], 250 ignore_errors=True, cwd=dest_dir) 251 merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], 252 ignore_errors=True, cwd=dest_dir) 253 merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], 254 ignore_errors=True, cwd=dest_dir) 255 # Only try to commit the makefiles if something has actually changed. 256 if _ModifiedFilesInIndex(dest_dir): 257 merge_common.GetCommandStdout( 258 ['git', 'commit', '-m', 259 'Update makefiles after merge of Chromium at r%s\n\n%s' % 260 (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir) 261 262 263def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): 264 """Returns true if git's index contains any changes.""" 265 status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], 266 cwd=cwd) 267 return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None 268 269 270def _GenerateNoticeFile(svn_revision): 271 """Generates and commits a NOTICE file containing code licenses. 272 273 This covers all third-party code (from Android's perspective) that lives in 274 the Chromium tree. 275 276 Args: 277 svn_revision: The SVN revision for the main Chromium repository. 278 """ 279 logging.debug('Regenerating NOTICE file ...') 280 281 contents = webview_licenses.GenerateNoticeFile() 282 283 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: 284 f.write(contents) 285 merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) 286 # Only try to commit the NOTICE update if the file has actually changed. 287 if _ModifiedFilesInIndex(): 288 merge_common.GetCommandStdout([ 289 'git', 'commit', '-m', 290 'Update NOTICE file after merge of Chromium at r%s\n\n%s' 291 % (svn_revision, AUTOGEN_MESSAGE)]) 292 293 294def _GenerateLastChange(svn_revision): 295 """Write a build/util/LASTCHANGE file containing the current revision. 296 297 The revision number is compiled into the binary at build time from this file. 298 299 Args: 300 svn_revision: The SVN revision for the main Chromium repository. 301 Raises: 302 MergeError: if the svn revision could not be found in the repository. 303 """ 304 logging.debug('Updating LASTCHANGE ...') 305 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 306 'w') as f: 307 f.write('LASTCHANGE=%s\n' % svn_revision) 308 merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) 309 if _ModifiedFilesInIndex(): 310 merge_common.GetCommandStdout([ 311 'git', 'commit', '-m', 312 'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s' 313 % (svn_revision, AUTOGEN_MESSAGE)]) 314 315 316def _GetSVNRevisionAndSHA1(git_url, git_branch, svn_revision): 317 logging.debug('Getting SVN revision and SHA1 ...') 318 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, 319 git_branch + ':cached_upstream']) 320 if svn_revision is None: 321 # Fetch LKGR from upstream. 322 with contextlib.closing( 323 urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr: 324 svn_revision = lkgr.read() 325 output = merge_common.GetCommandStdout([ 326 'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision, 327 '--format=%H', 'cached_upstream']) 328 if not output: 329 raise merge_common.MergeError('Revision %s not found in git repo.' % 330 svn_revision) 331 # The log grep will sometimes match reverts/reapplies of commits. We take the 332 # oldest (last) match because the first time it appears in history is 333 # overwhelmingly likely to be the correct commit. 334 sha1 = output.split()[-1] 335 return (svn_revision, sha1) 336 337 338def Snapshot(svn_revision, unattended): 339 """Takes a snapshot of the Chromium tree and merges it into Android. 340 341 Android makefiles and a top-level NOTICE file are generated and committed 342 after the merge. 343 344 Args: 345 svn_revision: The SVN revision in the Chromium repository to merge from. 346 unattended: Run in unattended mode. 347 348 Returns: 349 True if new commits were merged; False if no new commits were present. 350 """ 351 git_url = 'http://chromium.googlesource.com/chromium/src.git' 352 git_branch = 'git-svn' 353 (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_url, git_branch, 354 svn_revision) 355 if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', 356 'HEAD..' + root_sha1]): 357 logging.info('No new commits to merge from %s branch %s at r%s (%s)', 358 git_url, git_branch, svn_revision, root_sha1) 359 return False 360 361 logging.info('Snapshotting Chromium from %s branch %s at r%s (%s)', 362 git_url, git_branch, svn_revision, root_sha1) 363 364 # 1. Merge, accounting for excluded directories 365 _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended) 366 367 # 2. Generate Android NOTICE file 368 _GenerateNoticeFile(svn_revision) 369 370 # 3. Generate LASTCHANGE file 371 _GenerateLastChange(svn_revision) 372 373 # 4. Generate Android makefiles 374 _GenerateMakefiles(svn_revision) 375 376 return True 377 378 379def Push(): 380 """Push the finished snapshot to the Android repository.""" 381 merge_common.PushToServer('merge-from-chromium', 'master-chromium', 382 'master-chromium-merge') 383 384 385def main(): 386 parser = optparse.OptionParser(usage='%prog [options]') 387 parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' 388 'Chromium SVN revision and merges it into this repository. ' 389 'Paths marked as excluded for license reasons are removed ' 390 'as part of the merge. Also generates Android makefiles and ' 391 'generates a top-level NOTICE file suitable for use in the ' 392 'Android build.') 393 parser.add_option( 394 '', '--svn_revision', 395 default=None, 396 help=('Merge to the specified chromium SVN revision, rather than using ' 397 'the current LKGR.')) 398 parser.add_option( 399 '', '--push', 400 default=False, action='store_true', 401 help=('Push the result of a previous merge to the server.')) 402 parser.add_option( 403 '', '--unattended', 404 default=False, action='store_true', 405 help=('Run in unattended mode.')) 406 (options, args) = parser.parse_args() 407 if args: 408 parser.print_help() 409 return 1 410 411 if 'ANDROID_BUILD_TOP' not in os.environ: 412 print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' 413 return 1 414 415 logging.basicConfig(format='%(message)s', level=logging.DEBUG, 416 stream=sys.stdout) 417 418 if options.push: 419 Push() 420 else: 421 Snapshot(options.svn_revision, options.unattended) 422 423 return 0 424 425if __name__ == '__main__': 426 sys.exit(main()) 427