1"""\
2Logic for control file generation.
3"""
4
5__author__ = 'showard@google.com (Steve Howard)'
6
7import re, os
8
9import common
10from autotest_lib.frontend.afe import model_logic
11import frontend.settings
12
13AUTOTEST_DIR = os.path.abspath(os.path.join(
14    os.path.dirname(frontend.settings.__file__), '..'))
15
16EMPTY_TEMPLATE = 'def step_init():\n'
17
18CLIENT_KERNEL_TEMPLATE = """\
19kernel_list = %(client_kernel_list)s
20
21def step_init():
22    for kernel_info in kernel_list:
23        job.next_step(boot_kernel, kernel_info)
24        job.next_step(step_test, kernel_info['version'])
25    if len(kernel_list) > 1:
26        job.use_sequence_number = True  # include run numbers in directory names
27
28
29def boot_kernel(kernel_info):
30    # remove kernels (and associated data) not referenced by the bootloader
31    for host in job.hosts:
32        host.cleanup_kernels()
33
34    testkernel = job.kernel(kernel_info['version'])
35    if kernel_info['config_file']:
36        testkernel.config(kernel_info['config_file'])
37    testkernel.build()
38    testkernel.install()
39
40    cmdline = ' '.join((kernel_info.get('cmdline', ''), '%(kernel_args)s'))
41    testkernel.boot(args=cmdline)
42
43
44def step_test(kernel_version):
45    global kernel
46    kernel = kernel_version  # Set the global in case anyone is using it.
47    if len(kernel_list) > 1:
48        # this is local to a machine, safe to assume there's only one host
49        host, = job.hosts
50        job.automatic_test_tag = host.get_kernel_ver()
51"""
52
53SERVER_KERNEL_TEMPLATE = """\
54kernel_list = %%(server_kernel_list)s
55kernel_install_control = \"""
56%s    pass
57\"""
58
59from autotest_lib.client.common_lib import error
60
61at = autotest.Autotest()
62
63%%(upload_config_func)s
64def install_kernel(machine, kernel_info):
65    host = hosts.create_host(machine)
66    at.install(host=host)
67    %%(call_upload_config)s
68    at.run(kernel_install_control %%%%
69           {'client_kernel_list': repr([kernel_info])}, host=host)
70
71
72num_machines_required = len(machines)
73if len(machines) > 4:
74    # Allow a large multi-host tests to proceed despite a couple of hosts
75    # failing to properly install the desired kernel (exclude those hosts).
76    # TODO(gps): Figure out how to get and use SYNC_COUNT here.  It is defined
77    # within some control files and will end up inside of stepN functions below.
78    num_machines_required = len(machines) - 2
79
80
81def step_init():
82    # a host object we use solely for the purpose of finding out the booted
83    # kernel version, we use machines[0] since we already check that the same
84    # kernel has been booted on all machines
85    if len(kernel_list) > 1:
86        kernel_host = hosts.create_host(machines[0])
87
88    for kernel_info in kernel_list:
89        func = lambda machine: install_kernel(machine, kernel_info)
90        good_machines = job.parallel_on_machines(func, machines)
91        if len(good_machines) < num_machines_required:
92            raise error.TestError(
93                    "kernel installed on only %%%%d of %%%%d machines."
94                    %%%% (len(good_machines), num_machines_required))
95
96        # Replace the machines list that step_test() will use with the
97        # ones that successfully installed the kernel.
98        machines[:] = good_machines
99
100        # have server_job.run_test() automatically add the kernel version as
101        # a suffix to the test name otherwise we cannot run the same test on
102        # different kernel versions
103        if len(kernel_list) > 1:
104            job.automatic_test_tag = kernel_host.get_kernel_ver()
105        step_test()
106
107
108def step_test():
109""" % CLIENT_KERNEL_TEMPLATE
110
111CLIENT_STEP_TEMPLATE = "    job.next_step('step%d')\n"
112SERVER_STEP_TEMPLATE = '    step%d()\n'
113
114UPLOAD_CONFIG_FUNC = """
115def upload_kernel_config(host, kernel_info):
116    \"""
117    If the kernel_info['config_file'] is a URL it will be downloaded
118    locally and then uploaded to the client and a copy of the original
119    dictionary with the new path to the config file will be returned.
120    If the config file is not a URL the function returns the original
121    dictionary.
122    \"""
123    import os
124    from autotest_lib.client.common_lib import autotemp, utils
125
126    config_orig = kernel_info.get('config_file')
127
128    # if the file is not an URL then we assume it's a local client path
129    if not config_orig or not utils.is_url(config_orig):
130        return kernel_info
131
132    # download it locally (on the server) and send it to the client
133    config_tmp = autotemp.tempfile('kernel_config_upload', dir=job.tmpdir)
134    try:
135        utils.urlretrieve(config_orig, config_tmp.name)
136        config_new = os.path.join(host.get_autodir(), 'tmp',
137                                  os.path.basename(config_orig))
138        host.send_file(config_tmp.name, config_new)
139    finally:
140        config_tmp.clean()
141
142    return dict(kernel_info, config_file=config_new)
143
144"""
145
146CALL_UPLOAD_CONFIG = 'kernel_info = upload_kernel_config(host, kernel_info)'
147
148
149def kernel_config_file(kernel, platform):
150    if (not kernel.endswith('.rpm') and platform and
151        platform.kernel_config):
152        return platform.kernel_config
153    return None
154
155
156def read_control_file(test):
157    control_file = open(os.path.join(AUTOTEST_DIR, test.path))
158    control_contents = control_file.read()
159    control_file.close()
160    return control_contents
161
162
163def get_kernel_stanza(kernel_list, platform=None, kernel_args='',
164                      is_server=False, upload_kernel_config=False):
165
166    template_args = {'kernel_args' : kernel_args}
167
168    # add 'config_file' keys to the kernel_info dictionaries
169    new_kernel_list = []
170    for kernel_info in kernel_list:
171        if kernel_info.get('config_file'):
172            # already got a config file from the user
173            new_kernel_info = kernel_info
174        else:
175            config_file = kernel_config_file(kernel_info['version'], platform)
176            new_kernel_info = dict(kernel_info, config_file=config_file)
177
178        new_kernel_list.append(new_kernel_info)
179
180    if is_server:
181        template = SERVER_KERNEL_TEMPLATE
182        # leave client_kernel_list as a placeholder
183        template_args['client_kernel_list'] = '%(client_kernel_list)s'
184        template_args['server_kernel_list'] = repr(new_kernel_list)
185
186        if upload_kernel_config:
187            template_args['call_upload_config'] = CALL_UPLOAD_CONFIG
188            template_args['upload_config_func'] = UPLOAD_CONFIG_FUNC
189        else:
190            template_args['call_upload_config'] = ''
191            template_args['upload_config_func'] = ''
192    else:
193        template = CLIENT_KERNEL_TEMPLATE
194        template_args['client_kernel_list'] = repr(new_kernel_list)
195
196    return template % template_args
197
198
199def add_boilerplate_to_nested_steps(lines):
200    # Look for a line that begins with 'def step_init():' while
201    # being flexible on spacing.  If it's found, this will be
202    # a nested set of steps, so add magic to make it work.
203    # See client/bin/job.py's step_engine for more info.
204    if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines):
205        lines += '\nreturn locals() '
206        lines += '# Boilerplate magic for nested sets of steps'
207    return lines
208
209
210def format_step(item, lines):
211    lines = indent_text(lines, '    ')
212    lines = 'def step%d():\n%s' % (item, lines)
213    return lines
214
215
216def get_tests_stanza(tests, is_server, prepend=None, append=None,
217                     client_control_file=''):
218    """ Constructs the control file test step code from a list of tests.
219
220    @param tests A sequence of test control files to run.
221    @param is_server bool, Is this a server side test?
222    @param prepend A list of steps to prepend to each client test.
223        Defaults to [].
224    @param append A list of steps to append to each client test.
225        Defaults to [].
226    @param client_control_file If specified, use this text as the body of a
227        final client control file to run after tests.  is_server must be False.
228
229    @returns The control file test code to be run.
230    """
231    assert not (client_control_file and is_server)
232    if not prepend:
233        prepend = []
234    if not append:
235        append = []
236    raw_control_files = [read_control_file(test) for test in tests]
237    return _get_tests_stanza(raw_control_files, is_server, prepend, append,
238                             client_control_file=client_control_file)
239
240
241def _get_tests_stanza(raw_control_files, is_server, prepend, append,
242                      client_control_file=''):
243    """
244    Implements the common parts of get_test_stanza.
245
246    A site_control_file that wants to implement its own get_tests_stanza
247    likely wants to call this in the end.
248
249    @param raw_control_files A list of raw control file data to be combined
250        into a single control file.
251    @param is_server bool, Is this a server side test?
252    @param prepend A list of steps to prepend to each client test.
253    @param append A list of steps to append to each client test.
254    @param client_control_file If specified, use this text as the body of a
255        final client control file to append to raw_control_files after fixups.
256
257    @returns The combined mega control file.
258    """
259    if client_control_file:
260        # 'return locals()' is always appended incase the user forgot, it
261        # is necessary to allow for nested step engine execution to work.
262        raw_control_files.append(client_control_file + '\nreturn locals()')
263    raw_steps = prepend + [add_boilerplate_to_nested_steps(step)
264                           for step in raw_control_files] + append
265    steps = [format_step(index, step)
266             for index, step in enumerate(raw_steps)]
267    if is_server:
268        step_template = SERVER_STEP_TEMPLATE
269        footer = '\n\nstep_init()\n'
270    else:
271        step_template = CLIENT_STEP_TEMPLATE
272        footer = ''
273
274    header = ''.join(step_template % i for i in xrange(len(steps)))
275    return header + '\n' + '\n\n'.join(steps) + footer
276
277
278def indent_text(text, indent):
279    """Indent given lines of python code avoiding indenting multiline
280    quoted content (only for triple " and ' quoting for now)."""
281    regex = re.compile('(\\\\*)("""|\'\'\')')
282
283    res = []
284    in_quote = None
285    for line in text.splitlines():
286        # if not within a multinline quote indent the line contents
287        if in_quote:
288            res.append(line)
289        else:
290            res.append(indent + line)
291
292        while line:
293            match = regex.search(line)
294            if match:
295                # for an even number of backslashes before the triple quote
296                if len(match.group(1)) % 2 == 0:
297                    if not in_quote:
298                        in_quote = match.group(2)[0]
299                    elif in_quote == match.group(2)[0]:
300                        # if we found a matching end triple quote
301                        in_quote = None
302                line = line[match.end():]
303            else:
304                break
305
306    return '\n'.join(res)
307
308
309def _get_profiler_commands(profilers, is_server, profile_only):
310    prepend, append = [], []
311    if profile_only is not None:
312        prepend.append("job.default_profile_only = %r" % profile_only)
313    for profiler in profilers:
314        prepend.append("job.profilers.add('%s')" % profiler.name)
315        append.append("job.profilers.delete('%s')" % profiler.name)
316    return prepend, append
317
318
319def _sanity_check_generate_control(is_server, client_control_file, kernels,
320                                   upload_kernel_config):
321    """
322    Sanity check some of the parameters to generate_control().
323
324    This exists as its own function so that site_control_file may call it as
325    well from its own generate_control().
326
327    @raises ValidationError if any of the parameters do not make sense.
328    """
329    if is_server and client_control_file:
330        raise model_logic.ValidationError(
331                {'tests' : 'You cannot run server tests at the same time '
332                 'as directly supplying a client-side control file.'})
333
334    if kernels:
335        # make sure that kernel is a list of dictionarions with at least
336        # the 'version' key in them
337        kernel_error = model_logic.ValidationError(
338                {'kernel': 'The kernel parameter must be a sequence of '
339                 'dictionaries containing at least the "version" key '
340                 '(got: %r)' % kernels})
341        try:
342            iter(kernels)
343        except TypeError:
344            raise kernel_error
345        for kernel_info in kernels:
346            if (not isinstance(kernel_info, dict) or
347                    'version' not in kernel_info):
348                raise kernel_error
349
350        if upload_kernel_config and not is_server:
351            raise model_logic.ValidationError(
352                    {'upload_kernel_config': 'Cannot use upload_kernel_config '
353                                             'with client side tests'})
354
355
356def generate_control(tests, kernels=None, platform=None, is_server=False,
357                     profilers=(), client_control_file='', profile_only=None,
358                     upload_kernel_config=False):
359    """
360    Generate a control file for a sequence of tests.
361
362    @param tests A sequence of test control files to run.
363    @param kernels A sequence of kernel info dictionaries configuring which
364            kernels to boot for this job and other options for them
365    @param platform A platform object with a kernel_config attribute.
366    @param is_server bool, Is this a server control file rather than a client?
367    @param profilers A list of profiler objects to enable during the tests.
368    @param client_control_file Contents of a client control file to run as the
369            last test after everything in tests.  Requires is_server=False.
370    @param profile_only bool, should this control file run all tests in
371            profile_only mode by default
372    @param upload_kernel_config: if enabled it will generate server control
373            file code that uploads the kernel config file to the client and
374            tells the client of the new (local) path when compiling the kernel;
375            the tests must be server side tests
376
377    @returns The control file text as a string.
378    """
379    _sanity_check_generate_control(is_server=is_server, kernels=kernels,
380                                   client_control_file=client_control_file,
381                                   upload_kernel_config=upload_kernel_config)
382
383    control_file_text = ''
384    if kernels:
385        control_file_text = get_kernel_stanza(
386                kernels, platform, is_server=is_server,
387                upload_kernel_config=upload_kernel_config)
388    else:
389        control_file_text = EMPTY_TEMPLATE
390
391    prepend, append = _get_profiler_commands(profilers, is_server, profile_only)
392
393    control_file_text += get_tests_stanza(tests, is_server, prepend, append,
394                                          client_control_file)
395    return control_file_text
396