1#!/usr/bin/python2
2
3# Copyright 2014 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""Skia's Chromium DEPS roll script.
9
10This script:
11- searches through the last N Skia git commits to find out the hash that is
12  associated with the SVN revision number.
13- creates a new branch in the Chromium tree, modifies the DEPS file to
14  point at the given Skia commit, commits, uploads to Rietveld, and
15  deletes the local copy of the branch.
16- creates a whitespace-only commit and uploads that to to Rietveld.
17- returns the Chromium tree to its previous state.
18
19To specify the location of the git executable, set the GIT_EXECUTABLE
20environment variable.
21
22Usage:
23  %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
24"""
25
26
27import optparse
28import os
29import re
30import shutil
31import subprocess
32import sys
33import tempfile
34
35import git_utils
36import misc_utils
37
38
39DEFAULT_BOTS_LIST = [
40    'android_clang_dbg',
41    'android_dbg',
42    'android_rel',
43    'cros_daisy',
44    'linux',
45    'linux_asan',
46    'linux_chromeos',
47    'linux_chromeos_asan',
48    'linux_chromium_gn_dbg',
49    'linux_gpu',
50    'linux_layout',
51    'linux_layout_rel',
52    'mac',
53    'mac_asan',
54    'mac_gpu',
55    'mac_layout',
56    'mac_layout_rel',
57    'win',
58    'win_gpu',
59    'win_layout',
60    'win_layout_rel',
61]
62
63
64class DepsRollConfig(object):
65    """Contains configuration options for this module.
66
67    Attributes:
68        git: (string) The git executable.
69        chromium_path: (string) path to a local chromium git repository.
70        save_branches: (boolean) iff false, delete temporary branches.
71        verbose: (boolean)  iff false, suppress the output from git-cl.
72        search_depth: (int) how far back to look for the revision.
73        skia_url: (string) Skia's git repository.
74        self.skip_cl_upload: (boolean)
75        self.cl_bot_list: (list of strings)
76    """
77
78    # pylint: disable=I0011,R0903,R0902
79    def __init__(self, options=None):
80        self.skia_url = 'https://skia.googlesource.com/skia.git'
81        self.revision_format = (
82            'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
83
84        self.git = git_utils.git_executable()
85
86        if not options:
87            options = DepsRollConfig.GetOptionParser()
88        # pylint: disable=I0011,E1103
89        self.verbose = options.verbose
90        self.vsp = misc_utils.VerboseSubprocess(self.verbose)
91        self.save_branches = not options.delete_branches
92        self.search_depth = options.search_depth
93        self.chromium_path = options.chromium_path
94        self.skip_cl_upload = options.skip_cl_upload
95        # Split and remove empty strigns from the bot list.
96        self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
97        self.skia_git_checkout_path = options.skia_git_path
98        self.default_branch_name = 'autogenerated_deps_roll_branch'
99        self.reviewers_list = ','.join([
100            # 'rmistry@google.com',
101            # 'reed@google.com',
102            # 'bsalomon@google.com',
103            # 'robertphillips@google.com',
104            ])
105        self.cc_list = ','.join([
106            # 'skia-team@google.com',
107            ])
108
109    @staticmethod
110    def GetOptionParser():
111        # pylint: disable=I0011,C0103
112        """Returns an optparse.OptionParser object.
113
114        Returns:
115            An optparse.OptionParser object.
116
117        Called by the main() function.
118        """
119        option_parser = optparse.OptionParser(usage=__doc__)
120        # Anyone using this script on a regular basis should set the
121        # CHROMIUM_CHECKOUT_PATH environment variable.
122        option_parser.add_option(
123            '-c', '--chromium_path', help='Path to local Chromium Git'
124            ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
125            ' if that environment variable is set.',
126            default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
127        option_parser.add_option(
128            '-r', '--revision', type='int', default=None,
129            help='The Skia SVN revision number, defaults to top of tree.')
130        option_parser.add_option(
131            '-g', '--git_hash', default=None,
132            help='A partial Skia Git hash.  Do not set this and revision.')
133
134        # Anyone using this script on a regular basis should set the
135        # SKIA_GIT_CHECKOUT_PATH environment variable.
136        option_parser.add_option(
137            '', '--skia_git_path',
138            help='Path of a pure-git Skia repository checkout.  If empty,'
139            ' a temporary will be cloned.  Defaults to SKIA_GIT_CHECKOUT'
140            '_PATH, if that environment variable is set.',
141            default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
142        option_parser.add_option(
143            '', '--search_depth', type='int', default=100,
144            help='How far back to look for the revision.')
145        option_parser.add_option(
146            '', '--delete_branches', help='Delete the temporary branches',
147            action='store_true', dest='delete_branches', default=False)
148        option_parser.add_option(
149            '', '--verbose', help='Do not suppress the output from `git cl`.',
150            action='store_true', dest='verbose', default=False)
151        option_parser.add_option(
152            '', '--skip_cl_upload', help='Skip the cl upload step; useful'
153            ' for testing.',
154            action='store_true', default=False)
155
156        default_bots_help = (
157            'Comma-separated list of bots, defaults to a list of %d bots.'
158            '  To skip `git cl try`, set this to an empty string.'
159            % len(DEFAULT_BOTS_LIST))
160        default_bots = ','.join(DEFAULT_BOTS_LIST)
161        option_parser.add_option(
162            '', '--bots', help=default_bots_help, default=default_bots)
163
164        return option_parser
165
166
167class DepsRollError(Exception):
168    """Exceptions specific to this module."""
169    pass
170
171
172def get_svn_revision(config, commit):
173    """Works in both git and git-svn. returns a string."""
174    svn_format = (
175        '(git-svn-id: [^@ ]+@|SVN changes up to revision |'
176        'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)')
177    svn_revision = misc_utils.ReSearch.search_within_output(
178        config.verbose, svn_format, None,
179        [config.git, 'log', '-n', '1', '--format=format:%B', commit])
180    if not svn_revision:
181        raise DepsRollError(
182            'Revision number missing from Chromium origin/master.')
183    return int(svn_revision)
184
185
186class SkiaGitCheckout(object):
187    """Class to create a temporary skia git checkout, if necessary.
188    """
189    # pylint: disable=I0011,R0903
190
191    def __init__(self, config, depth):
192        self._config = config
193        self._depth = depth
194        self._use_temp = None
195        self._original_cwd = None
196
197    def __enter__(self):
198        config = self._config
199        git = config.git
200        skia_dir = None
201        self._original_cwd = os.getcwd()
202        if config.skia_git_checkout_path:
203            if config.skia_git_checkout_path != os.curdir:
204                skia_dir = config.skia_git_checkout_path
205                ## Update origin/master if needed.
206                if self._config.verbose:
207                    print '~~$', 'cd', skia_dir
208                os.chdir(skia_dir)
209            config.vsp.check_call([git, 'fetch', '-q', 'origin'])
210            self._use_temp = None
211        else:
212            skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
213            self._use_temp = skia_dir
214            try:
215                os.chdir(skia_dir)
216                config.vsp.check_call(
217                    [git, 'clone', '-q', '--depth=%d' % self._depth,
218                     '--single-branch', config.skia_url, '.'])
219            except (OSError, subprocess.CalledProcessError) as error:
220                shutil.rmtree(skia_dir)
221                raise error
222
223    def __exit__(self, etype, value, traceback):
224        if self._config.skia_git_checkout_path != os.curdir:
225            if self._config.verbose:
226                print '~~$', 'cd', self._original_cwd
227            os.chdir(self._original_cwd)
228        if self._use_temp:
229            shutil.rmtree(self._use_temp)
230
231
232def revision_and_hash(config):
233    """Finds revision number and git hash of origin/master in the Skia tree.
234
235    Args:
236        config: (roll_deps.DepsRollConfig) object containing options.
237
238    Returns:
239        A tuple (revision, hash)
240            revision: (int) SVN revision number.
241            git_hash: (string) full Git commit hash.
242
243    Raises:
244        roll_deps.DepsRollError: if the revision can't be found.
245        OSError: failed to execute git or git-cl.
246        subprocess.CalledProcessError: git returned unexpected status.
247    """
248    with SkiaGitCheckout(config, 1):
249        revision = get_svn_revision(config, 'origin/master')
250        git_hash = config.vsp.strip_output(
251            [config.git, 'show-ref', 'origin/master', '--hash'])
252        if not git_hash:
253            raise DepsRollError('Git hash can not be found.')
254    return revision, git_hash
255
256
257def revision_and_hash_from_revision(config, revision):
258    """Finds revision number and git hash of a commit in the Skia tree.
259
260    Args:
261        config: (roll_deps.DepsRollConfig) object containing options.
262        revision: (int) SVN revision number.
263
264    Returns:
265        A tuple (revision, hash)
266            revision: (int) SVN revision number.
267            git_hash: (string) full Git commit hash.
268
269    Raises:
270        roll_deps.DepsRollError: if the revision can't be found.
271        OSError: failed to execute git or git-cl.
272        subprocess.CalledProcessError: git returned unexpected status.
273    """
274    with SkiaGitCheckout(config, config.search_depth):
275        revision_regex = config.revision_format % revision
276        git_hash = config.vsp.strip_output(
277            [config.git, 'log', '--grep', revision_regex,
278             '--format=format:%H', 'origin/master'])
279        if not git_hash:
280            raise DepsRollError('Git hash can not be found.')
281    return revision, git_hash
282
283
284def revision_and_hash_from_partial(config, partial_hash):
285    """Returns the SVN revision number and full git hash.
286
287    Args:
288        config: (roll_deps.DepsRollConfig) object containing options.
289        partial_hash: (string) Partial git commit hash.
290
291    Returns:
292        A tuple (revision, hash)
293            revision: (int) SVN revision number.
294            git_hash: (string) full Git commit hash.
295
296    Raises:
297        roll_deps.DepsRollError: if the revision can't be found.
298        OSError: failed to execute git or git-cl.
299        subprocess.CalledProcessError: git returned unexpected status.
300    """
301    with SkiaGitCheckout(config, config.search_depth):
302        git_hash = config.vsp.strip_output(
303            ['git', 'log', '-n', '1', '--format=format:%H', partial_hash])
304        if not git_hash:
305            raise DepsRollError('Partial Git hash can not be found.')
306        revision = get_svn_revision(config, git_hash)
307    return revision, git_hash
308
309
310def change_skia_deps(revision, git_hash, depspath):
311    """Update the DEPS file.
312
313    Modify the skia_revision and skia_hash entries in the given DEPS file.
314
315    Args:
316        revision: (int) Skia SVN revision.
317        git_hash: (string) Skia Git hash.
318        depspath: (string) path to DEPS file.
319    """
320    temp_file = tempfile.NamedTemporaryFile(delete=False,
321                                            prefix='skia_DEPS_ROLL_tmp_')
322    try:
323        deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
324        deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')
325
326        deps_regex_rev_repl = '"skia_revision": "%d",' % revision
327        deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash
328
329        with open(depspath, 'r') as input_stream:
330            for line in input_stream:
331                line = deps_regex_rev.sub(deps_regex_rev_repl, line)
332                line = deps_regex_hash.sub(deps_regex_hash_repl, line)
333                temp_file.write(line)
334    finally:
335        temp_file.close()
336    shutil.move(temp_file.name, depspath)
337
338
339def git_cl_uploader(config, message, file_list):
340    """Create a commit in the current git branch; upload via git-cl.
341
342    Assumes that you are already on the branch you want to be on.
343
344    Args:
345        config: (roll_deps.DepsRollConfig) object containing options.
346        message: (string) the commit message, can be multiline.
347        file_list: (list of strings) list of filenames to pass to `git add`.
348
349    Returns:
350        The output of `git cl issue`, if not config.skip_cl_upload, else ''.
351    """
352
353    git, vsp = config.git, config.vsp
354    svn_info = str(get_svn_revision(config, 'HEAD'))
355
356    for filename in file_list:
357        assert os.path.exists(filename)
358        vsp.check_call([git, 'add', filename])
359
360    vsp.check_call([git, 'commit', '-q', '-m', message])
361
362    git_cl = [git, 'cl', 'upload', '-f',
363              '--bypass-hooks', '--bypass-watchlists']
364    if config.cc_list:
365        git_cl.append('--cc=%s' % config.cc_list)
366    if config.reviewers_list:
367        git_cl.append('--reviewers=%s' % config.reviewers_list)
368
369    git_try = [
370        git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info]
371    git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)])
372
373    branch_name = git_utils.git_branch_name(vsp.verbose)
374
375    if config.skip_cl_upload:
376        space = '   '
377        print 'You should call:'
378        print '%scd %s' % (space, os.getcwd())
379        misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name])
380        misc_utils.print_subprocess_args(space, git_cl)
381        if config.cl_bot_list:
382            misc_utils.print_subprocess_args(space, git_try)
383        print
384        return ''
385    else:
386        vsp.check_call(git_cl)
387        issue = vsp.strip_output([git, 'cl', 'issue'])
388        if config.cl_bot_list:
389            vsp.check_call(git_try)
390        return issue
391
392
393def roll_deps(config, revision, git_hash):
394    """Upload changed DEPS and a whitespace change.
395
396    Given the correct git_hash, create two Reitveld issues.
397
398    Args:
399        config: (roll_deps.DepsRollConfig) object containing options.
400        revision: (int) Skia SVN revision.
401        git_hash: (string) Skia Git hash.
402
403    Returns:
404        a tuple containing textual description of the two issues.
405
406    Raises:
407        OSError: failed to execute git or git-cl.
408        subprocess.CalledProcessError: git returned unexpected status.
409    """
410
411    git = config.git
412    with misc_utils.ChangeDir(config.chromium_path, config.verbose):
413        config.vsp.check_call([git, 'fetch', '-q', 'origin'])
414
415        old_revision = misc_utils.ReSearch.search_within_output(
416            config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
417            [git, 'show', 'origin/master:DEPS'])
418        assert old_revision
419        if revision == int(old_revision):
420            print 'DEPS is up to date!'
421            return (None, None)
422
423        master_hash = config.vsp.strip_output(
424            [git, 'show-ref', 'origin/master', '--hash'])
425        master_revision = get_svn_revision(config, 'origin/master')
426
427        # master_hash[8] gives each whitespace CL a unique name.
428        if config.save_branches:
429            branch = 'control_%s' % master_hash[:8]
430        else:
431            branch = None
432        message = ('whitespace change %s\n\n'
433                   'Chromium base revision: %d / %s\n\n'
434                   'This CL was created by Skia\'s roll_deps.py script.\n'
435                  ) % (master_hash[:8], master_revision, master_hash[:8])
436        with git_utils.ChangeGitBranch(branch, 'origin/master',
437                                       config.verbose):
438            branch = git_utils.git_branch_name(config.vsp.verbose)
439
440            with open('build/whitespace_file.txt', 'a') as output_stream:
441                output_stream.write('\nCONTROL\n')
442
443            whitespace_cl = git_cl_uploader(
444                config, message, ['build/whitespace_file.txt'])
445
446            control_url = misc_utils.ReSearch.search_within_string(
447                whitespace_cl, '(?P<return>https?://[^) ]+)', '?')
448            if config.save_branches:
449                whitespace_cl = '%s\n    branch: %s' % (whitespace_cl, branch)
450
451        if config.save_branches:
452            branch = 'roll_%d_%s' % (revision, master_hash[:8])
453        else:
454            branch = None
455        message = (
456            'roll skia DEPS to %d\n\n'
457            'Chromium base revision: %d / %s\n'
458            'Old Skia revision: %s\n'
459            'New Skia revision: %d\n'
460            'Control CL: %s\n\n'
461            'This CL was created by Skia\'s roll_deps.py script.\n\n'
462            'Bypassing commit queue trybots:\n'
463            'NOTRY=true\n'
464            % (revision, master_revision, master_hash[:8],
465               old_revision, revision, control_url))
466        with git_utils.ChangeGitBranch(branch, 'origin/master',
467                                       config.verbose):
468            branch = git_utils.git_branch_name(config.vsp.verbose)
469
470            change_skia_deps(revision, git_hash, 'DEPS')
471            deps_cl = git_cl_uploader(config, message, ['DEPS'])
472            if config.save_branches:
473                deps_cl = '%s\n    branch: %s' % (deps_cl, branch)
474
475        return deps_cl, whitespace_cl
476
477
478def find_hash_and_roll_deps(config, revision=None, partial_hash=None):
479    """Call find_hash_from_revision() and roll_deps().
480
481    The calls to git will be verbose on standard output.  After a
482    successful upload of both issues, print links to the new
483    codereview issues.
484
485    Args:
486        config: (roll_deps.DepsRollConfig) object containing options.
487        revision: (int or None) the Skia SVN revision number or None
488            to use the tip of the tree.
489        partial_hash: (string or None) a partial pure-git Skia commit
490            hash.  Don't pass both partial_hash and revision.
491
492    Raises:
493        roll_deps.DepsRollError: if the revision can't be found.
494        OSError: failed to execute git or git-cl.
495        subprocess.CalledProcessError: git returned unexpected status.
496    """
497
498    if revision and partial_hash:
499        raise DepsRollError('Pass revision or partial_hash, not both.')
500
501    if partial_hash:
502        revision, git_hash = revision_and_hash_from_partial(
503            config, partial_hash)
504    elif revision:
505        revision, git_hash = revision_and_hash_from_revision(config, revision)
506    else:
507        revision, git_hash = revision_and_hash(config)
508
509    print 'revision=%r\nhash=%r\n' % (revision, git_hash)
510
511    deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)
512
513    if deps_issue and whitespace_issue:
514        print 'DEPS roll:\n    %s\n' % deps_issue
515        print 'Whitespace change:\n    %s\n' % whitespace_issue
516    else:
517        print >> sys.stderr, 'No issues created.'
518
519
520def main(args):
521    """main function; see module-level docstring and GetOptionParser help.
522
523    Args:
524        args: sys.argv[1:]-type argument list.
525    """
526    option_parser = DepsRollConfig.GetOptionParser()
527    options = option_parser.parse_args(args)[0]
528
529    if not options.chromium_path:
530        option_parser.error('Must specify chromium_path.')
531    if not os.path.isdir(options.chromium_path):
532        option_parser.error('chromium_path must be a directory.')
533
534    if not git_utils.git_executable():
535        option_parser.error('Invalid git executable.')
536
537    config = DepsRollConfig(options)
538    find_hash_and_roll_deps(config, options.revision, options.git_hash)
539
540
541if __name__ == '__main__':
542    main(sys.argv[1:])
543
544