1#!/usr/bin/python
2
3# Copyright (c) 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to automate the process of deploying to production.
8
9Example usage of this script:
10  1. Update both autotest and chromite to the lastest commit that has passed
11     the test instance.
12     $ ./site_utils/automated_deploy.py
13  2. Skip updating a repo, e.g. autotest
14     $ ./site_utils/automated_deploy.py --skip_autotest
15  3. Update a given repo to a specific commit
16     $ ./site_utils/automated_deploy.py --autotest_hash='1234'
17"""
18
19import argparse
20import os
21import re
22import sys
23import subprocess
24
25import common
26from autotest_lib.client.common_lib import revision_control
27from autotest_lib.site_utils.lib import infra
28
29AUTOTEST_DIR = common.autotest_dir
30GIT_URL = {'autotest':
31           'https://chromium.googlesource.com/chromiumos/third_party/autotest',
32           'chromite':
33           'https://chromium.googlesource.com/chromiumos/chromite'}
34PROD_BRANCH = 'prod'
35MASTER_AFE = 'cautotest'
36NOTIFY_GROUP = 'chromeos-infra-discuss@google.com'
37
38# CIPD packages whose prod refs should be updated.
39_CIPD_PACKAGES = (
40        'chromiumos/infra/lucifer',
41        'chromiumos/infra/tast-cmd',
42        'chromiumos/infra/tast-remote-tests-cros',
43)
44
45
46class AutoDeployException(Exception):
47    """Raised when any deploy step fails."""
48
49
50def parse_arguments():
51    """Parse command line arguments.
52
53    @returns An argparse.Namespace populated with argument values.
54    """
55    parser = argparse.ArgumentParser(
56            description=('Command to update prod branch for autotest, chromite '
57                         'repos. Then deploy new changes to all lab servers.'))
58    parser.add_argument('--skip_autotest', action='store_true', default=False,
59            help='Skip updating autotest prod branch. Default is False.')
60    parser.add_argument('--skip_chromite', action='store_true', default=False,
61            help='Skip updating chromite prod branch. Default is False.')
62    parser.add_argument('--force_update', action='store_true', default=False,
63            help=('Force a deployment without updating both autotest and '
64                  'chromite prod branch'))
65    parser.add_argument('--autotest_hash', type=str, default=None,
66            help='Update autotest prod branch to the given hash. If it is not'
67                 ' specified, autotest prod branch will be rebased to '
68                 'prod-next branch, which is the latest commit that has '
69                 'passed our test instance.')
70    parser.add_argument('--chromite_hash', type=str, default=None,
71            help='Same as autotest_hash option.')
72
73    results = parser.parse_args(sys.argv[1:])
74
75    # Verify the validity of the options.
76    if ((results.skip_autotest and results.autotest_hash) or
77        (results.skip_chromite and results.chromite_hash)):
78        parser.print_help()
79        print 'Cannot specify skip_* and *_hash options at the same time.'
80        sys.exit(1)
81    if results.force_update:
82      results.skip_autotest = True
83      results.skip_chromite = True
84    return results
85
86
87def clone_prod_branch(repo):
88    """Method to clone the prod branch for a given repo under /tmp/ dir.
89
90    @param repo: Name of the git repo to be cloned.
91
92    @returns path to the cloned repo.
93    @raises subprocess.CalledProcessError on a command failure.
94    @raised revision_control.GitCloneError when git clone fails.
95    """
96    repo_dir = '/tmp/%s' % repo
97    print 'Cloning %s prod branch under %s' % (repo, repo_dir)
98    if os.path.exists(repo_dir):
99        infra.local_runner('rm -rf %s' % repo_dir)
100    git_repo = revision_control.GitRepo(repo_dir, GIT_URL[repo])
101    git_repo.clone(remote_branch=PROD_BRANCH)
102    print 'Successfully cloned %s prod branch' % repo
103    return repo_dir
104
105
106def update_prod_branch(repo, repo_dir, hash_to_rebase):
107    """Method to update the prod branch of the given repo to the given hash.
108
109    @param repo: Name of the git repo to be updated.
110    @param repo_dir: path to the cloned repo.
111    @param hash_to_rebase: Hash to rebase the prod branch to. If it is None,
112                           prod branch will rebase to prod-next branch.
113
114    @returns the range of the pushed commits as a string. E.g 123...345. If the
115        prod branch is already up-to-date, return None.
116    @raises subprocess.CalledProcessError on a command failure.
117    """
118    with infra.chdir(repo_dir):
119        print 'Updating %s prod branch.' % repo
120        rebase_to = hash_to_rebase if hash_to_rebase else 'origin/prod-next'
121        # Check whether prod branch is already up-to-date, which means there is
122        # no changes since last push.
123        print 'Detecting new changes since last push...'
124        diff = infra.local_runner('git log prod..%s --oneline' % rebase_to,
125                                  stream_output=True)
126        if diff:
127            print 'Find new changes, will update prod branch...'
128            infra.local_runner('git rebase %s prod' % rebase_to,
129                               stream_output=True)
130            result = infra.local_runner('git push origin prod',
131                                        stream_output=True)
132            print 'Successfully pushed %s prod branch!\n' % repo
133
134            # Get the pushed commit range, which is used to get pushed commits
135            # using git log E.g. 123..456, then run git log --oneline 123..456.
136            grep = re.search('(\w)*\.\.(\w)*', result)
137
138            if not grep:
139                raise AutoDeployException(
140                    'Fail to get pushed commits for repo %s from git log: %s' %
141                    (repo, result))
142            return grep.group(0)
143        else:
144            print 'No new %s changes found since last push.' % repo
145            return None
146
147
148def get_pushed_commits(repo, repo_dir, pushed_commits_range):
149    """Method to get the pushed commits.
150
151    @param repo: Name of the updated git repo.
152    @param repo_dir: path to the cloned repo.
153    @param pushed_commits_range: The range of the pushed commits. E.g 123...345
154    @return: the commits that are pushed to prod branch. The format likes this:
155             "git log --oneline A...B | grep autotest
156              A xxxx
157              B xxxx"
158    @raises subprocess.CalledProcessError on a command failure.
159    """
160    print 'Getting pushed CLs for %s repo.' % repo
161    if not pushed_commits_range:
162        return '\n%s:\nNo new changes since last push.' % repo
163
164    with infra.chdir(repo_dir):
165        get_commits_cmd = 'git log --oneline %s' % pushed_commits_range
166
167        pushed_commits = infra.local_runner(
168                get_commits_cmd, stream_output=True)
169        if repo == 'autotest':
170            autotest_commits = ''
171            for cl in pushed_commits.splitlines():
172                if 'autotest' in cl:
173                    autotest_commits += '%s\n' % cl
174
175            pushed_commits = autotest_commits
176
177        print 'Successfully got pushed CLs for %s repo!\n' % repo
178        displayed_cmd = get_commits_cmd
179        if repo == 'autotest':
180          displayed_cmd += ' | grep autotest'
181        return '\n%s:\n%s\n%s\n' % (repo, displayed_cmd, pushed_commits)
182
183
184def kick_off_deploy():
185    """Method to kick off deploy script to deploy changes to lab servers.
186
187    @raises subprocess.CalledProcessError on a repo command failure.
188    """
189    print 'Start deploying changes to all lab servers...'
190    with infra.chdir(AUTOTEST_DIR):
191        # Then kick off the deploy script.
192        deploy_cmd = ('runlocalssh ./site_utils/deploy_server.py -x --afe=%s' %
193                      MASTER_AFE)
194        infra.local_runner(deploy_cmd, stream_output=True)
195        print 'Successfully deployed changes to all lab servers.'
196
197
198def main(args):
199    """Main entry"""
200    options = parse_arguments()
201    repos = dict()
202    if not options.skip_autotest:
203        repos.update({'autotest': options.autotest_hash})
204    if not options.skip_chromite:
205        repos.update({'chromite': options.chromite_hash})
206
207    print 'Moving CIPD prod refs to prod-next'
208    for pkg in _CIPD_PACKAGES:
209        subprocess.check_call(['cipd', 'set-ref', pkg, '-version', 'prod-next',
210                               '-ref', 'prod'])
211    try:
212        # update_log saves the git log of the updated repo.
213        update_log = ''
214        for repo, hash_to_rebase in repos.iteritems():
215            repo_dir = clone_prod_branch(repo)
216            push_commits_range = update_prod_branch(
217                repo, repo_dir, hash_to_rebase)
218            update_log += get_pushed_commits(repo, repo_dir, push_commits_range)
219
220        kick_off_deploy()
221    except revision_control.GitCloneError as e:
222        print 'Fail to clone prod branch. Error:\n%s\n' % e
223        raise
224    except subprocess.CalledProcessError as e:
225        print ('Deploy fails when running a subprocess cmd :\n%s\n'
226               'Below is the push log:\n%s\n' % (e.output, update_log))
227        raise
228    except Exception as e:
229        print 'Deploy fails with error:\n%s\nPush log:\n%s\n' % (e, update_log)
230        raise
231
232    # When deploy succeeds, print the update_log.
233    print ('Deploy succeeds!!! Below is the push log of the updated repo:\n%s'
234           'Please email this to %s.'% (update_log, NOTIFY_GROUP))
235
236
237if __name__ == '__main__':
238    sys.exit(main(sys.argv))
239