1#!/usr/bin/python
2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Runs on autotest servers from a cron job to self update them.
7
8This script is designed to run on all autotest servers to allow them to
9automatically self-update based on the manifests used to create their (existing)
10repos.
11"""
12
13from __future__ import print_function
14
15import ConfigParser
16import argparse
17import os
18import re
19import subprocess
20import sys
21import time
22
23import common
24
25from autotest_lib.client.common_lib import global_config
26
27# How long after restarting a service do we watch it to see if it's stable.
28SERVICE_STABILITY_TIMER = 120
29
30
31class DirtyTreeException(Exception):
32    """Raised when the tree has been modified in an unexpected way."""
33
34
35class UnknownCommandException(Exception):
36    """Raised when we try to run a command name with no associated command."""
37
38
39class UnstableServices(Exception):
40    """Raised if a service appears unstable after restart."""
41
42
43def strip_terminal_codes(text):
44    """This function removes all terminal formatting codes from a string.
45
46    @param text: String of text to cleanup.
47    @returns String with format codes removed.
48    """
49    ESC = '\x1b'
50    return re.sub(ESC+r'\[[^m]*m', '', text)
51
52
53def verify_repo_clean():
54    """This function verifies that the current repo is valid, and clean.
55
56    @raises DirtyTreeException if the repo is not clean.
57    @raises subprocess.CalledProcessError on a repo command failure.
58    """
59    out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT)
60    out = strip_terminal_codes(out).strip()
61
62    CLEAN_STATUS_OUTPUT = 'nothing to commit (working directory clean)'
63    if out != CLEAN_STATUS_OUTPUT:
64      raise DirtyTreeException(out)
65
66
67def repo_versions():
68    """This function collects the versions of all git repos in the general repo.
69
70    @returns A dictionary mapping project names to git hashes for HEAD.
71    @raises subprocess.CalledProcessError on a repo command failure.
72    """
73    cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h']
74    output = strip_terminal_codes(subprocess.check_output(cmd))
75
76    # The expected output format is:
77
78    # project chrome_build/
79    # /dir/holding/chrome_build
80    # 73dee9d
81    #
82    # project chrome_release/
83    # /dir/holding/chrome_release
84    # 9f3a5d8
85
86    lines = output.splitlines()
87
88    PROJECT_PREFIX = 'project '
89
90    project_heads = {}
91    for n in range(0, len(lines), 4):
92        project_line = lines[n]
93        project_dir = lines[n+1]
94        project_hash = lines[n+2]
95        # lines[n+3] is a blank line, but doesn't exist for the final block.
96
97        # Convert 'project chrome_build/' -> 'chrome_build'
98        assert project_line.startswith(PROJECT_PREFIX)
99        name = project_line[len(PROJECT_PREFIX):].rstrip('/')
100
101        project_heads[name] = (project_dir, project_hash)
102
103    return project_heads
104
105
106def repo_sync():
107    """Perform a repo sync.
108
109    @raises subprocess.CalledProcessError on a repo command failure.
110    """
111    subprocess.check_output(['repo', 'sync'])
112
113
114def discover_update_commands():
115    """Lookup the commands to run on this server.
116
117    These commonly come from shadow_config.ini, since they vary by server type.
118
119    @returns List of command names in string format.
120    """
121    try:
122        return global_config.global_config.get_config_value(
123                'UPDATE', 'commands', type=list)
124
125    except (ConfigParser.NoSectionError, global_config.ConfigError):
126        return []
127
128
129def discover_restart_services():
130    """Find the services that need restarting on the current server.
131
132    These commonly come from shadow_config.ini, since they vary by server type.
133
134    @returns List of service names in string format.
135    """
136    try:
137        # From shadow_config.ini, lookup which services to restart.
138        return global_config.global_config.get_config_value(
139                'UPDATE', 'services', type=list)
140
141    except (ConfigParser.NoSectionError, global_config.ConfigError):
142        return []
143
144
145def update_command(cmd_tag, dryrun=False):
146    """Restart a command.
147
148    The command name is looked up in global_config.ini to find the full command
149    to run, then it's executed.
150
151    @param cmd_tag: Which command to restart.
152    @param dryrun: If true print the command that would have been run.
153
154    @raises UnknownCommandException If cmd_tag can't be looked up.
155    @raises subprocess.CalledProcessError on a command failure.
156    """
157    # Lookup the list of commands to consider. They are intended to be
158    # in global_config.ini so that they can be shared everywhere.
159    cmds = dict(global_config.global_config.config.items(
160        'UPDATE_COMMANDS'))
161
162    if cmd_tag not in cmds:
163        raise UnknownCommandException(cmd_tag, cmds)
164
165    expanded_command = cmds[cmd_tag].replace('AUTOTEST_REPO',
166                                              common.autotest_dir)
167
168    print('Running: %s: %s' % (cmd_tag, expanded_command))
169    if dryrun:
170        print('Skip: %s' % expanded_command)
171    else:
172        try:
173            subprocess.check_output(expanded_command, shell=True,
174                                    stderr=subprocess.STDOUT)
175        except subprocess.CalledProcessError as e:
176            print('FAILED:')
177            print(e.output)
178            raise
179
180
181def restart_service(service_name, dryrun=False):
182    """Restart a service.
183
184    Restarts the standard service with "service <name> restart".
185
186    @param service_name: The name of the service to restart.
187    @param dryrun: Don't really run anything, just print out the command.
188
189    @raises subprocess.CalledProcessError on a command failure.
190    """
191    cmd = ['sudo', 'service', service_name, 'restart']
192    print('Restarting: %s' % service_name)
193    if dryrun:
194        print('Skip: %s' % ' '.join(cmd))
195    else:
196        subprocess.check_call(cmd)
197
198
199def service_status(service_name):
200    """Return the results "status <name>" for a given service.
201
202    This string is expected to contain the pid, and so to change is the service
203    is shutdown or restarted for any reason.
204
205    @param service_name: The name of the service to check on.
206
207    @returns The output of the external command.
208             Ex: autofs start/running, process 1931
209
210    @raises subprocess.CalledProcessError on a command failure.
211    """
212    return subprocess.check_output(['sudo', 'status', service_name])
213
214
215def restart_services(service_names, dryrun=False, skip_service_status=False):
216    """Restart services as needed for the current server type.
217
218    Restart the listed set of services, and watch to see if they are stable for
219    at least SERVICE_STABILITY_TIMER. It restarts all services quickly,
220    waits for that delay, then verifies the status of all of them.
221
222    @param service_names: The list of service to restart and monitor.
223    @param dryrun: Don't really restart the service, just print out the command.
224    @param skip_service_status: Set to True to skip service status check.
225                                Default is False.
226
227    @raises subprocess.CalledProcessError on a command failure.
228    @raises UnstableServices if any services are unstable after restart.
229    """
230    service_statuses = {}
231
232    if dryrun:
233        for name in service_names:
234            restart_service(name, dryrun=True)
235        return
236
237    # Restart each, and record the status (including pid).
238    for name in service_names:
239        restart_service(name)
240        service_statuses[name] = service_status(name)
241
242    # Skip service status check if --skip-service-status is specified. Used for
243    # servers in backup status.
244    if skip_service_status:
245        print('--skip-service-status is specified, skip checking services.')
246        return
247
248    # Wait for a while to let the services settle.
249    time.sleep(SERVICE_STABILITY_TIMER)
250
251    # Look for any services that changed status.
252    unstable_services = [n for n in service_names
253                         if service_status(n) != service_statuses[n]]
254
255    # Report any services having issues.
256    if unstable_services:
257        raise UnstableServices(unstable_services)
258
259
260def run_deploy_actions(dryrun=False, skip_service_status=False):
261    """Run arbitrary update commands specified in global.ini.
262
263    @param dryrun: Don't really restart the service, just print out the command.
264    @param skip_service_status: Set to True to skip service status check.
265                                Default is False.
266
267    @raises subprocess.CalledProcessError on a command failure.
268    @raises UnstableServices if any services are unstable after restart.
269    """
270    cmds = discover_update_commands()
271    if cmds:
272        print('Running update commands:', ', '.join(cmds))
273        for cmd in cmds:
274            update_command(cmd, dryrun=dryrun)
275
276    services = discover_restart_services()
277    if services:
278        print('Restarting Services:', ', '.join(services))
279        restart_services(services, dryrun=dryrun,
280                         skip_service_status=skip_service_status)
281
282
283def report_changes(versions_before, versions_after):
284    """Produce a report describing what changed in all repos.
285
286    @param versions_before: Results of repo_versions() from before the update.
287    @param versions_after: Results of repo_versions() from after the update.
288
289    @returns string containing a human friendly changes report.
290    """
291    result = []
292
293    if versions_after:
294        for project in sorted(set(versions_before.keys() + versions_after.keys())):
295            result.append('%s:' % project)
296
297            _, before_hash = versions_before.get(project, (None, None))
298            after_dir, after_hash = versions_after.get(project, (None, None))
299
300            if project not in versions_before:
301                result.append('Added.')
302
303            elif project not in versions_after:
304                result.append('Removed.')
305
306            elif before_hash == after_hash:
307                result.append('No Change.')
308
309            else:
310                hashes = '%s..%s' % (before_hash, after_hash)
311                cmd = ['git', 'log', hashes, '--oneline']
312                out = subprocess.check_output(cmd, cwd=after_dir,
313                                              stderr=subprocess.STDOUT)
314                result.append(out.strip())
315
316            result.append('')
317    else:
318        for project in sorted(versions_before.keys()):
319            _, before_hash = versions_before[project]
320            result.append('%s: %s' % (project, before_hash))
321        result.append('')
322
323    return '\n'.join(result)
324
325
326def parse_arguments(args):
327    """Parse command line arguments.
328
329    @param args: The command line arguments to parse. (ususally sys.argsv[1:])
330
331    @returns An argparse.Namespace populated with argument values.
332    """
333    parser = argparse.ArgumentParser(
334            description='Command to update an autotest server.')
335    parser.add_argument('--skip-verify', action='store_false',
336                        dest='verify', default=True,
337                        help='Disable verification of a clean repository.')
338    parser.add_argument('--skip-update', action='store_false',
339                        dest='update', default=True,
340                        help='Skip the repository source code update.')
341    parser.add_argument('--skip-actions', action='store_false',
342                        dest='actions', default=True,
343                        help='Skip the post update actions.')
344    parser.add_argument('--skip-report', action='store_false',
345                        dest='report', default=True,
346                        help='Skip the git version report.')
347    parser.add_argument('--actions-only', action='store_true',
348                        help='Run the post update actions (restart services).')
349    parser.add_argument('--dryrun', action='store_true',
350                        help='Don\'t actually run any commands, just log.')
351    parser.add_argument('--skip-service-status', action='store_true',
352                        help='Skip checking the service status.')
353
354    results = parser.parse_args(args)
355
356    if results.actions_only:
357        results.verify = False
358        results.update = False
359        results.report = False
360
361    # TODO(dgarrett): Make these behaviors support dryrun.
362    if results.dryrun:
363        results.verify = False
364        results.update = False
365
366    return results
367
368
369def main(args):
370    """Main method."""
371    os.chdir(common.autotest_dir)
372    global_config.global_config.parse_config_file()
373
374    behaviors = parse_arguments(args)
375
376    if behaviors.verify:
377        try:
378            print('Checking tree status:')
379            verify_repo_clean()
380            print('Clean.')
381        except DirtyTreeException as e:
382            print('Local tree is dirty, can\'t perform update safely.')
383            print()
384            print('repo status:')
385            print(e.args[0])
386            return 1
387
388    versions_before = repo_versions()
389    versions_after = {}
390
391    if behaviors.update:
392        print('Updating Repo.')
393        repo_sync()
394        versions_after = repo_versions()
395
396    if behaviors.actions:
397        try:
398            run_deploy_actions(
399                    dryrun=behaviors.dryrun,
400                    skip_service_status=behaviors.skip_service_status)
401        except UnstableServices as e:
402            print('The following services were not stable after '
403                  'the update:')
404            print(e.args[0])
405            return 1
406
407    if behaviors.report:
408        print('Changes:')
409        print(report_changes(versions_before, versions_after))
410
411
412if __name__ == '__main__':
413    sys.exit(main(sys.argv[1:]))
414