1#!/usr/bin/python 2# 3# Copyright (C) 2012 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Merge Chromium into the Android tree.""" 18 19import logging 20import optparse 21import os 22import re 23import sys 24 25import merge_common 26 27 28# We need to import this *after* merging from upstream to get the latest 29# version. Set it to none here to catch uses before it's imported. 30webview_licenses = None 31 32 33AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' 34SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master' 35 36 37def _ReadGitFile(sha1, path, git_url=None, git_branch=None): 38 """Reads a file from a (possibly remote) git project at a specific revision. 39 40 Args: 41 sha1: The SHA1 at which to read. 42 path: The relative path of the file to read. 43 git_url: The URL of the git server, if reading a remote project. 44 git_branch: The branch to fetch, if reading a remote project. 45 Returns: 46 The contents of the specified file. 47 """ 48 if git_url: 49 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch]) 50 return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)]) 51 52 53def _ParseDEPS(deps_content): 54 """Parses the DEPS file from Chromium and returns its contents. 55 56 Args: 57 deps_content: The contents of the DEPS file as text. 58 Returns: 59 A dictionary of the contents of DEPS at the specified revision 60 """ 61 62 class FromImpl(object): 63 """Used to implement the From syntax.""" 64 65 def __init__(self, module_name): 66 self.module_name = module_name 67 68 def __str__(self): 69 return 'From("%s")' % self.module_name 70 71 class _VarImpl(object): 72 73 def __init__(self, custom_vars, local_scope): 74 self._custom_vars = custom_vars 75 self._local_scope = local_scope 76 77 def Lookup(self, var_name): 78 """Implements the Var syntax.""" 79 if var_name in self._custom_vars: 80 return self._custom_vars[var_name] 81 elif var_name in self._local_scope.get('vars', {}): 82 return self._local_scope['vars'][var_name] 83 raise Exception('Var is not defined: %s' % var_name) 84 85 tmp_locals = {} 86 var = _VarImpl({}, tmp_locals) 87 tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} 88 exec(deps_content) in tmp_globals, tmp_locals # pylint: disable=W0122 89 return tmp_locals 90 91 92def _GetProjectMergeInfo(projects, deps_vars): 93 """Gets the git URL and SHA1 for each project based on DEPS. 94 95 Args: 96 projects: The list of projects to consider. 97 deps_vars: The dictionary of dependencies from DEPS. 98 Returns: 99 A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' 100 Raises: 101 TemporaryMergeError: if a project to be merged is not found in DEPS. 102 """ 103 deps_fallback_order = [ 104 deps_vars['deps'], 105 deps_vars['deps_os']['unix'], 106 deps_vars['deps_os']['android'], 107 ] 108 result = {} 109 for path in projects: 110 for deps in deps_fallback_order: 111 if path: 112 upstream_path = os.path.join('src', path) 113 else: 114 upstream_path = 'src' 115 url_plus_sha1 = deps.get(upstream_path) 116 if url_plus_sha1: 117 break 118 else: 119 raise merge_common.TemporaryMergeError( 120 'Could not find DEPS entry for project %s. This probably ' 121 'means that the project list in merge_from_chromium.py needs to be ' 122 'updated.' % path) 123 match = re.match('(.*?)@(.*)', url_plus_sha1) 124 url = match.group(1) 125 sha1 = match.group(2) 126 logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path) 127 result[path] = {'url': url, 'sha1': sha1} 128 return result 129 130 131def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): 132 """Merges each required Chromium project into the Android repository. 133 134 DEPS is consulted to determine which revision each project must be merged 135 at. Only a whitelist of required projects are merged. 136 137 Args: 138 version: The version to mention in generated commit messages. 139 root_sha1: The git hash to merge in the root repository. 140 target: The target branch to merge to. 141 unattended: Run in unattended mode. 142 buildspec_url: URL for buildspec repository, when merging a branch. 143 Returns: 144 The abbrev sha1 merged. It will be either |root_sha1| itself (when merging 145 chromium trunk) or the upstream sha1 of the release. 146 Raises: 147 TemporaryMergeError: If incompatibly licensed code is left after pruning. 148 """ 149 # The logic for this step lives here, in the Android tree, as it makes no 150 # sense for a Chromium tree to know about this merge. 151 152 if unattended: 153 branch_create_flag = '-B' 154 else: 155 branch_create_flag = '-b' 156 branch_name = 'merge-from-chromium-%s' % version 157 158 logging.debug('Parsing DEPS ...') 159 if root_sha1: 160 deps_content = _ReadGitFile(root_sha1, 'DEPS') 161 else: 162 # TODO(primiano): At some point the release branches will use DEPS as well, 163 # instead of .DEPS.git. Rename below when that day will come. 164 deps_content = _ReadGitFile('FETCH_HEAD', 165 'releases/' + version + '/.DEPS.git', 166 buildspec_url, 167 'master') 168 169 deps_vars = _ParseDEPS(deps_content) 170 171 merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, 172 deps_vars) 173 174 for path in merge_info: 175 # webkit needs special handling as we have a local mirror 176 local_mirrored = path == 'third_party/WebKit' 177 url = merge_info[path]['url'] 178 sha1 = merge_info[path]['sha1'] 179 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 180 if local_mirrored: 181 remote = 'history' 182 else: 183 remote = 'goog' 184 merge_common.GetCommandStdout(['git', 'checkout', 185 branch_create_flag, branch_name, 186 '-t', remote + '/' + target], 187 cwd=dest_dir) 188 if not local_mirrored or not root_sha1: 189 logging.debug('Fetching project %s at %s ...', path, sha1) 190 fetch_args = ['git', 'fetch', url, sha1] 191 merge_common.GetCommandStdout(fetch_args, cwd=dest_dir) 192 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], 193 cwd=dest_dir): 194 logging.debug('Merging project %s at %s ...', path, sha1) 195 # Merge conflicts make git merge return 1, so ignore errors 196 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], 197 cwd=dest_dir, ignore_errors=True) 198 merge_common.CheckNoConflictsAndCommitMerge( 199 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), 200 cwd=dest_dir, unattended=unattended) 201 else: 202 logging.debug('No new commits to merge in project %s', path) 203 204 # Handle root repository separately. 205 merge_common.GetCommandStdout(['git', 'checkout', 206 branch_create_flag, branch_name, 207 '-t', 'history/' + target]) 208 if not root_sha1: 209 merge_info = _GetProjectMergeInfo([''], deps_vars) 210 url = merge_info['']['url'] 211 merged_sha1 = merge_info['']['sha1'] 212 merge_common.GetCommandStdout(['git', 'fetch', url, merged_sha1]) 213 merged_sha1 = merge_common.Abbrev(merged_sha1) 214 merge_msg_version = '%s (%s)' % (version, merged_sha1) 215 else: 216 merge_msg_version = root_sha1 217 merged_sha1 = root_sha1 218 219 logging.debug('Merging Chromium at %s ...', merged_sha1) 220 # Merge conflicts make git merge return 1, so ignore errors 221 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', merged_sha1], 222 ignore_errors=True) 223 merge_common.CheckNoConflictsAndCommitMerge( 224 'Merge Chromium at %s\n\n%s' 225 % (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended) 226 227 logging.debug('Getting directories to exclude ...') 228 229 # We import this now that we have merged the latest version. 230 # It imports to a global in order that it can be used to generate NOTICE 231 # later. We also disable writing bytecode to keep the source tree clean. 232 sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', 233 'tools')) 234 sys.dont_write_bytecode = True 235 global webview_licenses # pylint: disable=W0602 236 import webview_licenses # pylint: disable=W0621,W0612,C6204 237 import known_issues # pylint: disable=C6204 238 239 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): 240 logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in 241 exclude_list)) 242 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 243 merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + 244 exclude_list, cwd=dest_dir) 245 if _ModifiedFilesInIndex(dest_dir): 246 merge_common.GetCommandStdout(['git', 'commit', '-m', 247 'Exclude unwanted directories'], 248 cwd=dest_dir) 249 assert(root_sha1 is None or root_sha1 == merged_sha1) 250 return merged_sha1 251 252 253def _CheckLicenses(): 254 """Check that no incompatibly licensed directories exist.""" 255 directories_left_over = webview_licenses.GetIncompatibleDirectories() 256 if directories_left_over: 257 raise merge_common.TemporaryMergeError( 258 'Incompatibly licensed directories remain: ' + 259 '\n'.join(directories_left_over)) 260 261 262def _GenerateMakefiles(version, unattended): 263 """Run gyp to generate the Android build system makefiles. 264 265 Args: 266 version: The version to mention in generated commit messages. 267 unattended: Run in unattended mode. 268 """ 269 logging.debug('Generating makefiles ...') 270 271 # TODO(torne): come up with a way to deal with hooks from DEPS properly 272 273 # TODO(torne): The .tmp files are generated by 274 # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source 275 # tree. We should avoid this, or at least use a more specific name to avoid 276 # accidentally removing or adding other files. 277 for path in merge_common.ALL_PROJECTS: 278 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 279 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', 280 'GypAndroid.*.mk', '*.target.*.mk', 281 '*.host.*.mk', '*.tmp'], cwd=dest_dir) 282 283 try: 284 merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all']) 285 except merge_common.MergeError as e: 286 if not unattended: 287 raise 288 else: 289 for path in merge_common.ALL_PROJECTS: 290 merge_common.GetCommandStdout( 291 ['git', 'reset', '--hard'], 292 cwd=os.path.join(merge_common.REPOSITORY_ROOT, path)) 293 raise merge_common.TemporaryMergeError('Makefile generation failed: ' + 294 str(e)) 295 296 for path in merge_common.ALL_PROJECTS: 297 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 298 # git add doesn't have an --ignore-unmatch so we have to do this instead: 299 merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], 300 ignore_errors=True, cwd=dest_dir) 301 merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], 302 ignore_errors=True, cwd=dest_dir) 303 merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], 304 ignore_errors=True, cwd=dest_dir) 305 merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], 306 ignore_errors=True, cwd=dest_dir) 307 # Only try to commit the makefiles if something has actually changed. 308 if _ModifiedFilesInIndex(dest_dir): 309 merge_common.GetCommandStdout( 310 ['git', 'commit', '-m', 311 'Update makefiles after merge of Chromium at %s\n\n%s' % 312 (version, AUTOGEN_MESSAGE)], cwd=dest_dir) 313 314 315def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): 316 """Returns true if git's index contains any changes.""" 317 status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], 318 cwd=cwd) 319 return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None 320 321 322def _GenerateNoticeFile(version): 323 """Generates and commits a NOTICE file containing code licenses. 324 325 This covers all third-party code (from Android's perspective) that lives in 326 the Chromium tree. 327 328 Args: 329 version: The version to mention in generated commit messages. 330 """ 331 logging.debug('Regenerating NOTICE file ...') 332 333 contents = webview_licenses.GenerateNoticeFile() 334 335 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: 336 f.write(contents) 337 merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) 338 # Only try to commit the NOTICE update if the file has actually changed. 339 if _ModifiedFilesInIndex(): 340 merge_common.GetCommandStdout([ 341 'git', 'commit', '-m', 342 'Update NOTICE file after merge of Chromium at %s\n\n%s' 343 % (version, AUTOGEN_MESSAGE)]) 344 345 346def _GenerateLastChange(version, root_sha1): 347 """Write a build/util/LASTCHANGE file containing the current revision. 348 349 The revision number is compiled into the binary at build time from this file. 350 351 Args: 352 version: The version to mention in generated commit messages. 353 root_sha1: The SHA1 of the main project (before the merge). 354 """ 355 logging.debug('Updating LASTCHANGE ...') 356 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 357 'w') as f: 358 f.write('LASTCHANGE=%s\n' % merge_common.Abbrev(root_sha1)) 359 merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) 360 logging.debug('Updating LASTCHANGE.blink ...') 361 with open(os.path.join(merge_common.REPOSITORY_ROOT, 362 'build/util/LASTCHANGE.blink'), 'w') as f: 363 f.write('LASTCHANGE=%s\n' % _GetBlinkRevision()) 364 merge_common.GetCommandStdout(['git', 'add', '-f', 365 'build/util/LASTCHANGE.blink']) 366 if _ModifiedFilesInIndex(): 367 merge_common.GetCommandStdout([ 368 'git', 'commit', '-m', 369 'Update LASTCHANGE file after merge of Chromium at %s\n\n%s' 370 % (version, AUTOGEN_MESSAGE)]) 371 372 373def GetHEAD(): 374 """Fetch the latest HEAD revision from the Chromium Git mirror. 375 376 Returns: 377 The latest HEAD revision (A Git abbrev SHA1). 378 """ 379 return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD') 380 381 382def _ParseSvnRevisionFromGitCommitMessage(commit_message): 383 return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message, 384 flags=re.MULTILINE).group(1) 385 386 387def _GetGitAbbrevSHA1(git_branch, revision): 388 """Returns an abbrev. SHA for the given revision (or branch, if HEAD).""" 389 assert revision 390 logging.debug('Getting Git revision for %s ...', revision) 391 392 upstream = git_branch if revision == 'HEAD' else revision 393 394 # Make sure the remote and the branch exist locally. 395 try: 396 merge_common.GetCommandStdout([ 397 'git', 'show-ref', '--verify', '--quiet', git_branch]) 398 except merge_common.CommandError: 399 raise merge_common.TemporaryMergeError( 400 'Cannot find the branch %s. Have you sync\'d master-chromium in this ' 401 'checkout?' % git_branch) 402 403 # Make sure the |upstream| Git object has been mirrored. 404 try: 405 merge_common.GetCommandStdout([ 406 'git', 'merge-base', '--is-ancestor', upstream, git_branch]) 407 except merge_common.CommandError: 408 raise merge_common.TemporaryMergeError( 409 'Upstream object (%s) not reachable from %s' % (upstream, git_branch)) 410 411 abbrev_sha = merge_common.Abbrev(merge_common.GetCommandStdout( 412 ['git', 'rev-list', '--max-count=1', upstream]).split()[0]) 413 return abbrev_sha 414 415 416def _GetBlinkRevision(): 417 # TODO(primiano): Switch to Git as soon as Blink gets migrated as well. 418 commit = merge_common.GetCommandStdout( 419 ['git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'], 420 cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit')) 421 return _ParseSvnRevisionFromGitCommitMessage(commit) 422 423 424def Snapshot(root_sha1, release, target, unattended, buildspec_url): 425 """Takes a snapshot of the Chromium tree and merges it into Android. 426 427 Android makefiles and a top-level NOTICE file are generated and committed 428 after the merge. 429 430 Args: 431 root_sha1: The abbrev sha1 in the Chromium git mirror to merge from. 432 release: The Chromium release version to merge from (e.g. "30.0.1599.20"). 433 Only one of root_sha1 and release should be specified. 434 target: The target branch to merge to. 435 unattended: Run in unattended mode. 436 buildspec_url: URL for buildspec repository, used when merging a release. 437 438 Returns: 439 True if new commits were merged; False if no new commits were present. 440 """ 441 if release: 442 root_sha1 = None 443 version = release 444 else: 445 root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1) 446 version = root_sha1 447 448 assert (root_sha1 is not None and len(root_sha1) > 6) or version == release 449 450 if root_sha1 and not merge_common.GetCommandStdout( 451 ['git', 'rev-list', '-1', 'HEAD..' + root_sha1]): 452 logging.info('No new commits to merge at %s (%s)', version, root_sha1) 453 return False 454 455 logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1) 456 457 # 1. Merge, accounting for excluded directories 458 merged_sha1 = _MergeProjects(version, root_sha1, target, unattended, 459 buildspec_url) 460 461 # 2. Generate Android makefiles 462 _GenerateMakefiles(version, unattended) 463 464 # 3. Check for incompatible licenses 465 _CheckLicenses() 466 467 # 4. Generate Android NOTICE file 468 _GenerateNoticeFile(version) 469 470 # 5. Generate LASTCHANGE file 471 _GenerateLastChange(version, merged_sha1) 472 473 return True 474 475 476def Push(version, target): 477 """Push the finished snapshot to the Android repository.""" 478 src = 'merge-from-chromium-%s' % version 479 # Use forced pushes ('+' prefix) for the temporary and archive branches in 480 # case they already got updated by a previous (possibly failed?) merge, but 481 # do not force push to the real master-chromium branch as this could erase 482 # downstream changes. 483 refspecs = ['%s:%s' % (src, target), 484 '+%s:refs/archive/chromium-%s' % (src, version)] 485 if target == 'master-chromium': 486 refspecs.insert(0, '+%s:master-chromium-merge' % src) 487 for refspec in refspecs: 488 logging.debug('Pushing to server (%s) ...', refspec) 489 for path in merge_common.ALL_PROJECTS: 490 if path in merge_common.PROJECTS_WITH_FLAT_HISTORY: 491 remote = 'history' 492 else: 493 remote = 'goog' 494 logging.debug('Pushing %s', path) 495 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 496 merge_common.GetCommandStdout(['git', 'push', remote, refspec], 497 cwd=dest_dir) 498 499 500def main(): 501 parser = optparse.OptionParser(usage='%prog [options]') 502 parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' 503 'Chromium Git revision and merges it into this repository. ' 504 'Paths marked as excluded for license reasons are removed ' 505 'as part of the merge. Also generates Android makefiles and ' 506 'generates a top-level NOTICE file suitable for use in the ' 507 'Android build.') 508 parser.add_option( 509 '', '--sha1', 510 default='HEAD', 511 help=('Merge to the specified chromium sha1 revision from ' + 512 SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.')) 513 parser.add_option( 514 '', '--release', 515 default=None, 516 help=('Merge to the specified chromium release buildspec (e.g., "30.0.' 517 '1599.20"). Only one of --sha1 and --release should be specified')) 518 parser.add_option( 519 '', '--buildspec_url', 520 default=None, 521 help=('Git URL for buildspec repository.')) 522 parser.add_option( 523 '', '--target', 524 default='master-chromium', metavar='BRANCH', 525 help=('Target branch to push to. Defaults to master-chromium.')) 526 parser.add_option( 527 '', '--push', 528 default=False, action='store_true', 529 help=('Push the result of a previous merge to the server. Note ' 530 '--sha1 must be given.')) 531 parser.add_option( 532 '', '--get_head', 533 default=False, action='store_true', 534 help=('Just print the current HEAD revision on stdout and exit.')) 535 parser.add_option( 536 '', '--unattended', 537 default=False, action='store_true', 538 help=('Run in unattended mode.')) 539 parser.add_option( 540 '', '--no_changes_exit', 541 default=0, type='int', 542 help=('Exit code to use if there are no changes to merge, for scripts.')) 543 (options, args) = parser.parse_args() 544 if args: 545 parser.print_help() 546 return 1 547 548 if 'ANDROID_BUILD_TOP' not in os.environ: 549 print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' 550 return 1 551 552 if os.environ.get('GYP_DEFINES'): 553 print >>sys.stderr, ( 554 'The environment is defining GYP_DEFINES (=%s). It will affect the ' 555 ' generated makefiles.' % os.environ['GYP_DEFINES']) 556 if not options.unattended and raw_input('Continue? [y/N]') != 'y': 557 return 1 558 559 logging.basicConfig(format='%(message)s', level=logging.DEBUG, 560 stream=sys.stdout) 561 562 if options.get_head: 563 logging.disable(logging.CRITICAL) # Prevent log messages 564 print GetHEAD() 565 elif options.push: 566 if options.release: 567 Push(options.release, options.target) 568 elif options.sha1: 569 Push(options.sha1, options.target) 570 else: 571 print >>sys.stderr, 'You need to pass the version to push.' 572 return 1 573 else: 574 if not Snapshot(options.sha1, options.release, options.target, 575 options.unattended, options.buildspec_url): 576 return options.no_changes_exit 577 578 return 0 579 580if __name__ == '__main__': 581 sys.exit(main()) 582