version_0.py revision d8561576404bcb7c3c56d1c691129ba285693d03 
1import os
2import re
3
4import common
5from autotest_lib.tko import models
6from autotest_lib.tko import status_lib
7from autotest_lib.tko import utils as tko_utils
8from autotest_lib.tko.parsers import base
9
10class NoHostnameError(Exception):
11    pass
12
13
14class BoardLabelError(Exception):
15    pass
16
17
18class job(models.job):
19    def __init__(self, dir):
20        job_dict = job.load_from_dir(dir)
21        super(job, self).__init__(dir, **job_dict)
22
23
24    @classmethod
25    def load_from_dir(cls, dir):
26        keyval = cls.read_keyval(dir)
27        tko_utils.dprint(str(keyval))
28
29        user = keyval.get("user", None)
30        label = keyval.get("label", None)
31        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
32        started_time = tko_utils.get_timestamp(keyval, "job_started")
33        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
34        machine = cls.determine_hostname(keyval, dir)
35        machine_group = cls.determine_machine_group(machine, dir)
36        machine_owner = keyval.get("owner", None)
37
38        aborted_by = keyval.get("aborted_by", None)
39        aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
40
41        return {"user": user, "label": label, "machine": machine,
42                "queued_time": queued_time, "started_time": started_time,
43                "finished_time": finished_time, "machine_owner": machine_owner,
44                "machine_group": machine_group, "aborted_by": aborted_by,
45                "aborted_on": aborted_at, "keyval_dict": keyval}
46
47
48    @classmethod
49    def determine_hostname(cls, keyval, job_dir):
50        host_group_name = keyval.get("host_group_name", None)
51        machine = keyval.get("hostname", "")
52        is_multimachine = "," in machine
53
54        # determine what hostname to use
55        if host_group_name:
56            if is_multimachine or not machine:
57                tko_utils.dprint("Using host_group_name %r instead of "
58                                 "machine name." % host_group_name)
59                machine = host_group_name
60        elif is_multimachine:
61            try:
62                machine = job.find_hostname(job_dir) # find a unique hostname
63            except NoHostnameError:
64                pass  # just use the comma-separated name
65
66        tko_utils.dprint("MACHINE NAME: %s" % machine)
67        return machine
68
69
70    @classmethod
71    def determine_machine_group(cls, hostname, job_dir):
72        machine_groups = set()
73        for individual_hostname in hostname.split(","):
74            host_keyval = models.test.parse_host_keyval(job_dir,
75                                                        individual_hostname)
76            if not host_keyval:
77                tko_utils.dprint('Unable to parse host keyval for %s'
78                                 % individual_hostname)
79            elif 'labels' in host_keyval:
80                # Use board label as machine group. This is to avoid the
81                # confusion of multiple boards mapping to the same platform in
82                # wmatrix. With this change, wmatrix will group tests with the
83                # same board, rather than the same platform.
84                labels = host_keyval['labels'].split(',')
85                board_labels = [l[8:] for l in labels
86                               if l.startswith('board%3A')]
87                if board_labels:
88                    # Testbeds have multiple boards so concat them into a
89                    # single string then add it to the machine_groups list.
90                    machine_groups.add(','.join(board_labels))
91                else:
92                    error = ('Failed to retrieve board label from host labels: '
93                             '%s' % host_keyval['labels'])
94                    tko_utils.dprint(error)
95                    raise BoardLabelError(error)
96            elif "platform" in host_keyval:
97                machine_groups.add(host_keyval["platform"])
98        machine_group = ",".join(sorted(machine_groups))
99        tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
100        return machine_group
101
102
103    @staticmethod
104    def find_hostname(path):
105        hostname = os.path.join(path, "sysinfo", "hostname")
106        try:
107            machine = open(hostname).readline().rstrip()
108            return machine
109        except Exception:
110            tko_utils.dprint("Could not read a hostname from "
111                             "sysinfo/hostname")
112
113        uname = os.path.join(path, "sysinfo", "uname_-a")
114        try:
115            machine = open(uname).readline().split()[1]
116            return machine
117        except Exception:
118            tko_utils.dprint("Could not read a hostname from "
119                             "sysinfo/uname_-a")
120
121        raise NoHostnameError("Unable to find a machine name")
122
123
124class kernel(models.kernel):
125    def __init__(self, job, verify_ident=None):
126        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
127        super(kernel, self).__init__(**kernel_dict)
128
129
130    @staticmethod
131    def load_from_dir(dir, verify_ident=None):
132        # try and load the booted kernel version
133        attributes = False
134        i = 1
135        build_dir = os.path.join(dir, "build")
136        while True:
137            if not os.path.exists(build_dir):
138                break
139            build_log = os.path.join(build_dir, "debug", "build_log")
140            attributes = kernel.load_from_build_log(build_log)
141            if attributes:
142                break
143            i += 1
144            build_dir = os.path.join(dir, "build.%d" % (i))
145
146        if not attributes:
147            if verify_ident:
148                base = verify_ident
149            else:
150                base = kernel.load_from_sysinfo(dir)
151            patches = []
152            hashes = []
153        else:
154            base, patches, hashes = attributes
155        tko_utils.dprint("kernel.__init__() found kernel version %s"
156                         % base)
157
158        # compute the kernel hash
159        if base == "UNKNOWN":
160            kernel_hash = "UNKNOWN"
161        else:
162            kernel_hash = kernel.compute_hash(base, hashes)
163
164        return {"base": base, "patches": patches,
165                "kernel_hash": kernel_hash}
166
167
168    @staticmethod
169    def load_from_sysinfo(path):
170        for subdir in ("reboot1", ""):
171            uname_path = os.path.join(path, "sysinfo", subdir,
172                                      "uname_-a")
173            if not os.path.exists(uname_path):
174                continue
175            uname = open(uname_path).readline().split()
176            return re.sub("-autotest$", "", uname[2])
177        return "UNKNOWN"
178
179
180    @staticmethod
181    def load_from_build_log(path):
182        if not os.path.exists(path):
183            return None
184
185        base, patches, hashes = "UNKNOWN", [], []
186        for line in file(path):
187            head, rest = line.split(": ", 1)
188            rest = rest.split()
189            if head == "BASE":
190                base = rest[0]
191            elif head == "PATCH":
192                patches.append(patch(*rest))
193                hashes.append(rest[2])
194        return base, patches, hashes
195
196
197class test(models.test):
198    def __init__(self, subdir, testname, status, reason, test_kernel,
199                 machine, started_time, finished_time, iterations,
200                 attributes, labels):
201        # for backwards compatibility with the original parser
202        # implementation, if there is no test version we need a NULL
203        # value to be used; also, if there is a version it should
204        # be terminated by a newline
205        if "version" in attributes:
206            attributes["version"] = str(attributes["version"])
207        else:
208            attributes["version"] = None
209
210        super(test, self).__init__(subdir, testname, status, reason,
211                                   test_kernel, machine, started_time,
212                                   finished_time, iterations,
213                                   attributes, labels)
214
215
216    @staticmethod
217    def load_iterations(keyval_path):
218        return iteration.load_from_keyval(keyval_path)
219
220
221class patch(models.patch):
222    def __init__(self, spec, reference, hash):
223        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
224        super(patch, self).__init__(spec, reference, hash)
225        self.spec = spec
226        self.reference = reference
227        self.hash = hash
228
229
230class iteration(models.iteration):
231    @staticmethod
232    def parse_line_into_dicts(line, attr_dict, perf_dict):
233        key, value = line.split("=", 1)
234        perf_dict[key] = value
235
236
237class status_line(object):
238    def __init__(self, indent, status, subdir, testname, reason,
239                 optional_fields):
240        # pull out the type & status of the line
241        if status == "START":
242            self.type = "START"
243            self.status = None
244        elif status.startswith("END "):
245            self.type = "END"
246            self.status = status[4:]
247        else:
248            self.type = "STATUS"
249            self.status = status
250        assert (self.status is None or
251                self.status in status_lib.statuses)
252
253        # save all the other parameters
254        self.indent = indent
255        self.subdir = self.parse_name(subdir)
256        self.testname = self.parse_name(testname)
257        self.reason = reason
258        self.optional_fields = optional_fields
259
260
261    @staticmethod
262    def parse_name(name):
263        if name == "----":
264            return None
265        return name
266
267
268    @staticmethod
269    def is_status_line(line):
270        return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
271
272
273    @classmethod
274    def parse_line(cls, line):
275        if not status_line.is_status_line(line):
276            return None
277        match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
278        if not match:
279            # A more useful error message than:
280            #  AttributeError: 'NoneType' object has no attribute 'groups'
281            # to help us debug WTF happens on occasion here.
282            raise RuntimeError("line %r could not be parsed." % line)
283        indent, line = match.groups()
284        indent = len(indent)
285
286        # split the line into the fixed and optional fields
287        parts = line.rstrip("\n").split("\t")
288
289        part_index = 3
290        status, subdir, testname = parts[0:part_index]
291
292        # all optional parts should be of the form "key=value". once we've found
293        # a non-matching part, treat it and the rest of the parts as the reason.
294        optional_fields = {}
295        while part_index < len(parts):
296            kv = re.search(r"^(\w+)=(.+)", parts[part_index])
297            if not kv:
298                break
299
300            optional_fields[kv.group(1)] = kv.group(2)
301            part_index += 1
302
303        reason = "\t".join(parts[part_index:])
304
305        # build up a new status_line and return it
306        return cls(indent, status, subdir, testname, reason,
307                   optional_fields)
308
309
310class parser(base.parser):
311    @staticmethod
312    def make_job(dir):
313        return job(dir)
314
315
316    def state_iterator(self, buffer):
317        new_tests = []
318        boot_count = 0
319        group_subdir = None
320        sought_level = 0
321        stack = status_lib.status_stack()
322        current_kernel = kernel(self.job)
323        boot_in_progress = False
324        alert_pending = None
325        started_time = None
326
327        while not self.finished or buffer.size():
328            # stop processing once the buffer is empty
329            if buffer.size() == 0:
330                yield new_tests
331                new_tests = []
332                continue
333
334            # parse the next line
335            line = buffer.get()
336            tko_utils.dprint('\nSTATUS: ' + line.strip())
337            line = status_line.parse_line(line)
338            if line is None:
339                tko_utils.dprint('non-status line, ignoring')
340                continue # ignore non-status lines
341
342            # have we hit the job start line?
343            if (line.type == "START" and not line.subdir and
344                not line.testname):
345                sought_level = 1
346                tko_utils.dprint("found job level start "
347                                 "marker, looking for level "
348                                 "1 groups now")
349                continue
350
351            # have we hit the job end line?
352            if (line.type == "END" and not line.subdir and
353                not line.testname):
354                tko_utils.dprint("found job level end "
355                                 "marker, looking for level "
356                                 "0 lines now")
357                sought_level = 0
358
359            # START line, just push another layer on to the stack
360            # and grab the start time if this is at the job level
361            # we're currently seeking
362            if line.type == "START":
363                group_subdir = None
364                stack.start()
365                if line.indent == sought_level:
366                    started_time = \
367                                 tko_utils.get_timestamp(
368                        line.optional_fields, "timestamp")
369                tko_utils.dprint("start line, ignoring")
370                continue
371            # otherwise, update the status on the stack
372            else:
373                tko_utils.dprint("GROPE_STATUS: %s" %
374                                 [stack.current_status(),
375                                  line.status, line.subdir,
376                                  line.testname, line.reason])
377                stack.update(line.status)
378
379            if line.status == "ALERT":
380                tko_utils.dprint("job level alert, recording")
381                alert_pending = line.reason
382                continue
383
384            # ignore Autotest.install => GOOD lines
385            if (line.testname == "Autotest.install" and
386                line.status == "GOOD"):
387                tko_utils.dprint("Successful Autotest "
388                                 "install, ignoring")
389                continue
390
391            # ignore END lines for a reboot group
392            if (line.testname == "reboot" and line.type == "END"):
393                tko_utils.dprint("reboot group, ignoring")
394                continue
395
396            # convert job-level ABORTs into a 'CLIENT_JOB' test, and
397            # ignore other job-level events
398            if line.testname is None:
399                if (line.status == "ABORT" and
400                    line.type != "END"):
401                    line.testname = "CLIENT_JOB"
402                else:
403                    tko_utils.dprint("job level event, "
404                                    "ignoring")
405                    continue
406
407            # use the group subdir for END lines
408            if line.type == "END":
409                line.subdir = group_subdir
410
411            # are we inside a block group?
412            if (line.indent != sought_level and
413                line.status != "ABORT" and
414                not line.testname.startswith('reboot.')):
415                if line.subdir:
416                    tko_utils.dprint("set group_subdir: "
417                                     + line.subdir)
418                    group_subdir = line.subdir
419                tko_utils.dprint("ignoring incorrect indent "
420                                 "level %d != %d," %
421                                 (line.indent, sought_level))
422                continue
423
424            # use the subdir as the testname, except for
425            # boot.* and kernel.* tests
426            if (line.testname is None or
427                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
428                              line.testname)):
429                if line.subdir and '.' in line.subdir:
430                    line.testname = line.subdir
431
432            # has a reboot started?
433            if line.testname == "reboot.start":
434                started_time = tko_utils.get_timestamp(
435                    line.optional_fields, "timestamp")
436                tko_utils.dprint("reboot start event, "
437                                 "ignoring")
438                boot_in_progress = True
439                continue
440
441            # has a reboot finished?
442            if line.testname == "reboot.verify":
443                line.testname = "boot.%d" % boot_count
444                tko_utils.dprint("reboot verified")
445                boot_in_progress = False
446                verify_ident = line.reason.strip()
447                current_kernel = kernel(self.job, verify_ident)
448                boot_count += 1
449
450            if alert_pending:
451                line.status = "ALERT"
452                line.reason = alert_pending
453                alert_pending = None
454
455            # create the actual test object
456            finished_time = tko_utils.get_timestamp(
457                line.optional_fields, "timestamp")
458            final_status = stack.end()
459            tko_utils.dprint("Adding: "
460                             "%s\nSubdir:%s\nTestname:%s\n%s" %
461                             (final_status, line.subdir,
462                              line.testname, line.reason))
463            new_test = test.parse_test(self.job, line.subdir,
464                                       line.testname,
465                                       final_status, line.reason,
466                                       current_kernel,
467                                       started_time,
468                                       finished_time)
469            started_time = None
470            new_tests.append(new_test)
471
472        # the job is finished, but we never came back from reboot
473        if boot_in_progress:
474            testname = "boot.%d" % boot_count
475            reason = "machine did not return from reboot"
476            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
477                              "Testname:%s\n%s")
478                             % (testname, reason))
479            new_test = test.parse_test(self.job, None, testname,
480                                       "ABORT", reason,
481                                       current_kernel, None, None)
482            new_tests.append(new_test)
483        yield new_tests
484