roll_deps.py revision 31fdb92995bbd49cc3b60a5d6b97fd20b0c0ef47
1#!/usr/bin/python2 2 3# Copyright 2014 Google Inc. 4# 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Skia's Chromium DEPS roll script. 9 10This script: 11- searches through the last N Skia git commits to find out the hash that is 12 associated with the SVN revision number. 13- creates a new branch in the Chromium tree, modifies the DEPS file to 14 point at the given Skia commit, commits, uploads to Rietveld, and 15 deletes the local copy of the branch. 16- creates a whitespace-only commit and uploads that to to Rietveld. 17- returns the Chromium tree to its previous state. 18 19Usage: 20 %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS] 21""" 22 23 24import optparse 25import os 26import re 27import shutil 28import subprocess 29from subprocess import check_call 30import sys 31import tempfile 32 33 34class DepsRollConfig(object): 35 """Contains configuration options for this module. 36 37 Attributes: 38 git: (string) The git executable. 39 chromium_path: (string) path to a local chromium git repository. 40 save_branches: (boolean) iff false, delete temporary branches. 41 verbose: (boolean) iff false, suppress the output from git-cl. 42 search_depth: (int) how far back to look for the revision. 43 skia_url: (string) Skia's git repository. 44 self.skip_cl_upload: (boolean) 45 self.cl_bot_list: (list of strings) 46 """ 47 48 # pylint: disable=I0011,R0903,R0902 49 def __init__(self, options=None): 50 self.skia_url = 'https://skia.googlesource.com/skia.git' 51 self.revision_format = ( 52 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') 53 54 if not options: 55 options = DepsRollConfig.GetOptionParser() 56 # pylint: disable=I0011,E1103 57 self.verbose = options.verbose 58 self.save_branches = options.save_branches 59 self.search_depth = options.search_depth 60 self.chromium_path = options.chromium_path 61 self.git = options.git_path 62 self.skip_cl_upload = options.skip_cl_upload 63 # Split and remove empty strigns from the bot list. 64 self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] 65 self.skia_git_checkout_path = options.skia_git_path 66 self.default_branch_name = 'autogenerated_deps_roll_branch' 67 68 @staticmethod 69 def GetOptionParser(): 70 # pylint: disable=I0011,C0103 71 """Returns an optparse.OptionParser object. 72 73 Returns: 74 An optparse.OptionParser object. 75 76 Called by the main() function. 77 """ 78 default_bots_list = [ 79 'android_clang_dbg', 80 'android_dbg', 81 'android_rel', 82 'cros_daisy', 83 'linux', 84 'linux_asan', 85 'linux_chromeos', 86 'linux_chromeos_asan', 87 'linux_gpu', 88 'linux_heapcheck', 89 'linux_layout', 90 'linux_layout_rel', 91 'mac', 92 'mac_asan', 93 'mac_gpu', 94 'mac_layout', 95 'mac_layout_rel', 96 'win', 97 'win_gpu', 98 'win_layout', 99 'win_layout_rel', 100 ] 101 102 option_parser = optparse.OptionParser(usage=__doc__) 103 # Anyone using this script on a regular basis should set the 104 # CHROMIUM_CHECKOUT_PATH environment variable. 105 option_parser.add_option( 106 '-c', '--chromium_path', help='Path to local Chromium Git' 107 ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH' 108 ' if that environment variable is set.', 109 default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) 110 option_parser.add_option( 111 '-r', '--revision', type='int', default=None, 112 help='The Skia SVN revision number, defaults to top of tree.') 113 # Anyone using this script on a regular basis should set the 114 # SKIA_GIT_CHECKOUT_PATH environment variable. 115 option_parser.add_option( 116 '', '--skia_git_path', 117 help='Path of a pure-git Skia repository checkout. If empty,' 118 ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' 119 '_PATH, if that environment variable is set.', 120 default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) 121 option_parser.add_option( 122 '', '--search_depth', type='int', default=100, 123 help='How far back to look for the revision.') 124 option_parser.add_option( 125 '', '--git_path', help='Git executable, defaults to "git".', 126 default='git') 127 option_parser.add_option( 128 '', '--save_branches', help='Save the temporary branches', 129 action='store_true', dest='save_branches', default=False) 130 option_parser.add_option( 131 '', '--verbose', help='Do not suppress the output from `git cl`.', 132 action='store_true', dest='verbose', default=False) 133 option_parser.add_option( 134 '', '--skip_cl_upload', help='Skip the cl upload step; useful' 135 ' for testing or with --save_branches.', 136 action='store_true', default=False) 137 138 default_bots_help = ( 139 'Comma-separated list of bots, defaults to a list of %d bots.' 140 ' To skip `git cl try`, set this to an empty string.' 141 % len(default_bots_list)) 142 default_bots = ','.join(default_bots_list) 143 option_parser.add_option( 144 '', '--bots', help=default_bots_help, default=default_bots) 145 146 return option_parser 147 148 149def test_git_executable(git_executable): 150 """Test the git executable. 151 152 Args: 153 git_executable: git executable path. 154 Returns: 155 True if test is successful. 156 """ 157 with open(os.devnull, 'w') as devnull: 158 try: 159 subprocess.call([git_executable, '--version'], stdout=devnull) 160 except (OSError,): 161 return False 162 return True 163 164 165class DepsRollError(Exception): 166 """Exceptions specific to this module.""" 167 pass 168 169 170def strip_output(*args, **kwargs): 171 """Wrap subprocess.check_output and str.strip(). 172 173 Pass the given arguments into subprocess.check_output() and return 174 the results, after stripping any excess whitespace. 175 176 Args: 177 *args: to be passed to subprocess.check_output() 178 **kwargs: to be passed to subprocess.check_output() 179 180 Returns: 181 The output of the process as a string without leading or 182 trailing whitespace. 183 Raises: 184 OSError or subprocess.CalledProcessError: raised by check_output. 185 """ 186 return str(subprocess.check_output(*args, **kwargs)).strip() 187 188 189def create_temp_skia_clone(config, depth): 190 """Clones Skia in a temp dir. 191 192 Args: 193 config: (roll_deps.DepsRollConfig) object containing options. 194 depth: (int) how far back to clone the tree. 195 Returns: 196 temporary directory path if succcessful. 197 Raises: 198 OSError, subprocess.CalledProcessError on failure. 199 """ 200 git = config.git 201 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') 202 try: 203 check_call( 204 [git, 'clone', '-q', '--depth=%d' % depth, 205 '--single-branch', config.skia_url, skia_dir]) 206 return skia_dir 207 except (OSError, subprocess.CalledProcessError) as error: 208 shutil.rmtree(skia_dir) 209 raise error 210 211 212def find_revision_and_hash(config, revision): 213 """Finds revision number and git hash of origin/master in the Skia tree. 214 215 Args: 216 config: (roll_deps.DepsRollConfig) object containing options. 217 revision: (int or None) SVN revision number. If None, use 218 tip-of-tree. 219 220 Returns: 221 A tuple (revision, hash) 222 revision: (int) SVN revision number. 223 hash: (string) full Git commit hash. 224 225 Raises: 226 roll_deps.DepsRollError: if the revision can't be found. 227 OSError: failed to execute git or git-cl. 228 subprocess.CalledProcessError: git returned unexpected status. 229 """ 230 git = config.git 231 use_temp = False 232 skia_dir = None 233 depth = 1 if (revision is None) else config.search_depth 234 try: 235 if config.skia_git_checkout_path: 236 skia_dir = config.skia_git_checkout_path 237 ## Update origin/master if needed. 238 check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir) 239 else: 240 skia_dir = create_temp_skia_clone(config, depth) 241 assert skia_dir 242 use_temp = True 243 244 if revision is None: 245 message = subprocess.check_output( 246 [git, 'log', '-n', '1', '--format=format:%B', 247 'origin/master'], cwd=skia_dir) 248 svn_format = ( 249 'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ') 250 search = re.search(svn_format, message) 251 if not search: 252 raise DepsRollError( 253 'Revision number missing from origin/master.') 254 revision = int(search.group(1)) 255 git_hash = strip_output( 256 [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir) 257 else: 258 revision_regex = config.revision_format % revision 259 git_hash = strip_output( 260 [git, 'log', '--grep', revision_regex, '--format=format:%H', 261 'origin/master'], cwd=skia_dir) 262 263 if revision < 0 or not git_hash: 264 raise DepsRollError('Git hash can not be found.') 265 return revision, git_hash 266 finally: 267 if use_temp: 268 shutil.rmtree(skia_dir) 269 270 271class GitBranchCLUpload(object): 272 """Class to manage git branches and git-cl-upload. 273 274 This class allows one to create a new branch in a repository based 275 off of origin/master, make changes to the tree inside the 276 with-block, upload that new branch to Rietveld, restore the original 277 tree state, and delete the local copy of the new branch. 278 279 See roll_deps() for an example of use. 280 281 Constructor Args: 282 config: (roll_deps.DepsRollConfig) object containing options. 283 message: (string) the commit message, can be multiline. 284 set_brach_name: (string or none) if not None, the name of the 285 branch to use. If None, then use a temporary branch that 286 will be deleted. 287 288 Attributes: 289 issue: a string describing the codereview issue, after __exit__ 290 has been called, othrwise, None. 291 292 Raises: 293 OSError: failed to execute git or git-cl. 294 subprocess.CalledProcessError: git returned unexpected status. 295 """ 296 # pylint: disable=I0011,R0903,R0902 297 298 def __init__(self, config, message, set_branch_name): 299 self._message = message 300 self._file_list = [] 301 self._branch_name = set_branch_name 302 self._stash = None 303 self._original_branch = None 304 self._config = config 305 self._svn_info = None 306 self.issue = None 307 308 def stage_for_commit(self, *paths): 309 """Calls `git add ...` on each argument. 310 311 Args: 312 *paths: (list of strings) list of filenames to pass to `git add`. 313 """ 314 self._file_list.extend(paths) 315 316 def __enter__(self): 317 git = self._config.git 318 def branch_exists(branch): 319 """Return true iff branch exists.""" 320 return 0 == subprocess.call( 321 [git, 'show-ref', '--quiet', branch]) 322 def has_diff(): 323 """Return true iff repository has uncommited changes.""" 324 return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD'])) 325 self._stash = has_diff() 326 if self._stash: 327 check_call([git, 'stash', 'save']) 328 try: 329 self._original_branch = strip_output( 330 [git, 'symbolic-ref', '--short', 'HEAD']) 331 except (subprocess.CalledProcessError,): 332 self._original_branch = strip_output( 333 [git, 'rev-parse', 'HEAD']) 334 335 if not self._branch_name: 336 self._branch_name = self._config.default_branch_name 337 338 if branch_exists(self._branch_name): 339 check_call([git, 'checkout', '-q', 'master']) 340 check_call([git, 'branch', '-q', '-D', self._branch_name]) 341 342 check_call( 343 [git, 'checkout', '-q', '-b', 344 self._branch_name, 'origin/master']) 345 346 svn_info = subprocess.check_output(['git', 'svn', 'info']) 347 svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info) 348 assert svn_info_search 349 self._svn_info = svn_info_search.group(1) 350 351 def __exit__(self, etype, value, traceback): 352 # pylint: disable=I0011,R0912 353 git = self._config.git 354 def quiet_check_call(*args, **kwargs): 355 """Call check_call, but pipe output to devnull.""" 356 with open(os.devnull, 'w') as devnull: 357 check_call(*args, stdout=devnull, **kwargs) 358 359 for filename in self._file_list: 360 assert os.path.exists(filename) 361 check_call([git, 'add', filename]) 362 check_call([git, 'commit', '-q', '-m', self._message]) 363 364 git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com', 365 '--bypass-hooks', '--bypass-watchlists'] 366 git_try = [git, 'cl', 'try', '--revision', self._svn_info] 367 git_try.extend([arg for bot in self._config.cl_bot_list 368 for arg in ('-b', bot)]) 369 370 if self._config.skip_cl_upload: 371 print ' '.join(git_cl) 372 print 373 if self._config.cl_bot_list: 374 print ' '.join(git_try) 375 print 376 self.issue = '' 377 else: 378 if self._config.verbose: 379 check_call(git_cl) 380 print 381 else: 382 quiet_check_call(git_cl) 383 self.issue = strip_output([git, 'cl', 'issue']) 384 if self._config.cl_bot_list: 385 if self._config.verbose: 386 check_call(git_try) 387 print 388 else: 389 quiet_check_call(git_try) 390 391 # deal with the aftermath of failed executions of this script. 392 if self._config.default_branch_name == self._original_branch: 393 self._original_branch = 'master' 394 check_call([git, 'checkout', '-q', self._original_branch]) 395 396 if self._config.default_branch_name == self._branch_name: 397 check_call([git, 'branch', '-q', '-D', self._branch_name]) 398 if self._stash: 399 check_call([git, 'stash', 'pop']) 400 401 402def change_skia_deps(revision, git_hash, depspath): 403 """Update the DEPS file. 404 405 Modify the skia_revision and skia_hash entries in the given DEPS file. 406 407 Args: 408 revision: (int) Skia SVN revision. 409 git_hash: (string) Skia Git hash. 410 depspath: (string) path to DEPS file. 411 """ 412 temp_file = tempfile.NamedTemporaryFile(delete=False, 413 prefix='skia_DEPS_ROLL_tmp_') 414 try: 415 deps_regex_rev = re.compile('"skia_revision": "[0-9]*",') 416 deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",') 417 418 deps_regex_rev_repl = '"skia_revision": "%d",' % revision 419 deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash 420 421 with open(depspath, 'r') as input_stream: 422 for line in input_stream: 423 line = deps_regex_rev.sub(deps_regex_rev_repl, line) 424 line = deps_regex_hash.sub(deps_regex_hash_repl, line) 425 temp_file.write(line) 426 finally: 427 temp_file.close() 428 shutil.move(temp_file.name, depspath) 429 430 431def branch_name(message): 432 """Return the first line of a commit message to be used as a branch name. 433 434 Args: 435 message: (string) 436 437 Returns: 438 A string derived from message suitable for a branch name. 439 """ 440 return message.lstrip().split('\n')[0].rstrip().replace(' ', '_') 441 442 443def roll_deps(config, revision, git_hash): 444 """Upload changed DEPS and a whitespace change. 445 446 Given the correct git_hash, create two Reitveld issues. 447 448 Args: 449 config: (roll_deps.DepsRollConfig) object containing options. 450 revision: (int) Skia SVN revision. 451 git_hash: (string) Skia Git hash. 452 453 Returns: 454 a tuple containing textual description of the two issues. 455 456 Raises: 457 OSError: failed to execute git or git-cl. 458 subprocess.CalledProcessError: git returned unexpected status. 459 """ 460 git = config.git 461 cwd = os.getcwd() 462 os.chdir(config.chromium_path) 463 try: 464 check_call([git, 'fetch', '-q', 'origin']) 465 master_hash = strip_output( 466 [git, 'show-ref', 'origin/master', '--hash']) 467 468 # master_hash[8] gives each whitespace CL a unique name. 469 message = ('whitespace change %s\n\nThis CL was created by' 470 ' Skia\'s roll_deps.py script.\n') % master_hash[:8] 471 branch = branch_name(message) if config.save_branches else None 472 473 codereview = GitBranchCLUpload(config, message, branch) 474 with codereview: 475 with open('build/whitespace_file.txt', 'a') as output_stream: 476 output_stream.write('\nCONTROL\n') 477 codereview.stage_for_commit('build/whitespace_file.txt') 478 whitespace_cl = codereview.issue 479 if branch: 480 whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) 481 control_url_match = re.search('https?://[^) ]+', codereview.issue) 482 if control_url_match: 483 message = ('roll skia DEPS to %d\n\nThis CL was created by' 484 ' Skia\'s roll_deps.py script.\n\ncontrol: %s' 485 % (revision, control_url_match.group(0))) 486 else: 487 message = ('roll skia DEPS to %d\n\nThis CL was created by' 488 ' Skia\'s roll_deps.py script.') % revision 489 branch = branch_name(message) if config.save_branches else None 490 codereview = GitBranchCLUpload(config, message, branch) 491 with codereview: 492 change_skia_deps(revision, git_hash, 'DEPS') 493 codereview.stage_for_commit('DEPS') 494 deps_cl = codereview.issue 495 if branch: 496 deps_cl = '%s\n branch: %s' % (deps_cl, branch) 497 498 return deps_cl, whitespace_cl 499 finally: 500 os.chdir(cwd) 501 502 503def find_hash_and_roll_deps(config, revision): 504 """Call find_hash_from_revision() and roll_deps(). 505 506 The calls to git will be verbose on standard output. After a 507 successful upload of both issues, print links to the new 508 codereview issues. 509 510 Args: 511 config: (roll_deps.DepsRollConfig) object containing options. 512 revision: (int or None) the Skia SVN revision number or None 513 to use the tip of the tree. 514 515 Raises: 516 roll_deps.DepsRollError: if the revision can't be found. 517 OSError: failed to execute git or git-cl. 518 subprocess.CalledProcessError: git returned unexpected status. 519 """ 520 revision, git_hash = find_revision_and_hash(config, revision) 521 522 print 'revision=%r\nhash=%r\n' % (revision, git_hash) 523 524 deps_issue, whitespace_issue = roll_deps(config, revision, git_hash) 525 526 print 'DEPS roll:\n %s\n' % deps_issue 527 print 'Whitespace change:\n %s\n' % whitespace_issue 528 529 530def main(args): 531 """main function; see module-level docstring and GetOptionParser help. 532 533 Args: 534 args: sys.argv[1:]-type argument list. 535 """ 536 option_parser = DepsRollConfig.GetOptionParser() 537 options = option_parser.parse_args(args)[0] 538 539 if not options.chromium_path: 540 option_parser.error('Must specify chromium_path.') 541 if not os.path.isdir(options.chromium_path): 542 option_parser.error('chromium_path must be a directory.') 543 if not test_git_executable(options.git_path): 544 option_parser.error('Invalid git executable.') 545 546 config = DepsRollConfig(options) 547 find_hash_and_roll_deps(config, options.revision) 548 549 550if __name__ == '__main__': 551 main(sys.argv[1:]) 552 553