version_0.py revision ba1fa66c641651bcf0d058e9f5af2819c2d2690a 
1import re, os
2
3from autotest_lib.client.common_lib import utils as common_utils
4from autotest_lib.tko import utils as tko_utils, models, status_lib
5from autotest_lib.tko.parsers import base
6
7
8class job(models.job):
9    def __init__(self, dir):
10        job_dict = job.load_from_dir(dir)
11        super(job, self).__init__(dir, **job_dict)
12
13    @staticmethod
14    def load_from_dir(dir):
15        try:
16            keyval = common_utils.read_keyval(dir)
17            tko_utils.dprint(str(keyval))
18        except Exception:
19            keyval = {}
20
21        user = keyval.get("user", None)
22        label = keyval.get("label", None)
23        machine = keyval.get("hostname", None)
24        if machine:
25            assert "," not in machine
26        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
27        started_time = tko_utils.get_timestamp(keyval, "job_started")
28        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
29        machine_owner = keyval.get("owner", None)
30
31        if not machine:
32            machine = job.find_hostname(dir)
33        tko_utils.dprint("MACHINE NAME: %s" % machine)
34
35        return {"user": user, "label": label, "machine": machine,
36                "queued_time": queued_time,
37                "started_time": started_time,
38                "finished_time": finished_time,
39                "machine_owner": machine_owner}
40
41
42    @staticmethod
43    def find_hostname(path):
44        hostname = os.path.join(path, "sysinfo", "hostname")
45        try:
46            machine = open(hostname).readline().rstrip()
47            return machine
48        except Exception:
49            tko_utils.dprint("Could not read a hostname from "
50                             "sysinfo/hostname")
51
52        uname = os.path.join(path, "sysinfo", "uname_-a")
53        try:
54            machine = open(uname).readline().split()[1]
55            return
56        except Exception:
57            tko_utils.dprint("Could not read a hostname from "
58                             "sysinfo/uname_-a")
59
60        raise Exception("Unable to find a machine name")
61
62
63class kernel(models.kernel):
64    def __init__(self, job, verify_ident=None):
65        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
66        super(kernel, self).__init__(**kernel_dict)
67
68
69    @staticmethod
70    def load_from_dir(dir, verify_ident=None):
71        # try and load the booted kernel version
72        build_log = os.path.join(dir, "build", "debug", "build_log")
73        attributes = kernel.load_from_build_log(build_log)
74        if not attributes:
75            if verify_ident:
76                base = verify_ident
77            else:
78                base = kernel.load_from_sysinfo(dir)
79            patches = []
80            hashes = []
81        else:
82            base, patches, hashes = attributes
83        tko_utils.dprint("kernel.__init__() found kernel version %s"
84                         % base)
85
86        # compute the kernel hash
87        if base == "UNKNOWN":
88            kernel_hash = "UNKNOWN"
89        else:
90            kernel_hash = kernel.compute_hash(base, hashes)
91
92        return {"base": base, "patches": patches,
93                "kernel_hash": kernel_hash}
94
95
96    @staticmethod
97    def load_from_sysinfo(path):
98        for subdir in ("reboot1", ""):
99            uname_path = os.path.join(path, "sysinfo", subdir,
100                                      "uname_-a")
101            if not os.path.exists(uname_path):
102                continue
103            uname = open(uname_path).readline().split()
104            return re.sub("-autotest$", "", uname[2])
105        return "UNKNOWN"
106
107
108    @staticmethod
109    def load_from_build_log(path):
110        if not os.path.exists(path):
111            return None
112
113        base, patches, hashes = "UNKNOWN", [], []
114        for line in file(path):
115            head, rest = line.split(": ", 1)
116            rest = rest.split()
117            if head == "BASE":
118                base = rest[0]
119            elif head == "PATCH":
120                patches.append(patch(*rest))
121                hashes.append(rest[2])
122        return base, patches, hashes
123
124
125class test(models.test):
126    def __init__(self, subdir, testname, status, reason, test_kernel,
127                 machine, started_time, finished_time, iterations,
128                 attributes):
129        # for backwards compatibility with the original parser
130        # implementation, if there is no test version we need a NULL
131        # value to be used; also, if there is a version it should
132        # be terminated by a newline
133        if "version" in attributes:
134            attributes["version"] = str(attributes["version"])
135        else:
136            attributes["version"] = None
137
138        super(test, self).__init__(subdir, testname, status, reason,
139                                   test_kernel, machine, started_time,
140                                   finished_time, iterations,
141                                   attributes)
142
143
144    @staticmethod
145    def load_iterations(keyval_path):
146        return iteration.load_from_keyval(keyval_path)
147
148
149class patch(models.patch):
150    def __init__(self, spec, reference, hash):
151        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
152        super(patch, self).__init__(spec, reference, hash)
153        self.spec = spec
154        self.reference = reference
155        self.hash = hash
156
157
158class iteration(models.iteration):
159    @staticmethod
160    def parse_line_into_dicts(line, attr_dict, perf_dict):
161        key, value = line.split("=", 1)
162        perf_dict[key] = value
163
164
165class status_line(object):
166    def __init__(self, indent, status, subdir, testname, reason,
167                 optional_fields):
168        # pull out the type & status of the line
169        if status == "START":
170            self.type = "START"
171            self.status = None
172        elif status.startswith("END "):
173            self.type = "END"
174            self.status = status[4:]
175        else:
176            self.type = "STATUS"
177            self.status = status
178        assert (self.status is None or
179                self.status in status_lib.statuses)
180
181        # save all the other parameters
182        self.indent = indent
183        self.subdir = self.parse_name(subdir)
184        self.testname = self.parse_name(testname)
185        self.reason = reason
186        self.optional_fields = optional_fields
187
188
189    @staticmethod
190    def parse_name(name):
191        if name == "----":
192            return None
193        return name
194
195
196    @staticmethod
197    def is_status_line(line):
198        return re.search(r"^\t*\S", line) is not None
199
200
201    @classmethod
202    def parse_line(cls, line):
203        if not status_line.is_status_line(line):
204            return None
205        indent, line = re.search(r"^(\t*)(.*)$", line).groups()
206        indent = len(indent)
207
208        # split the line into the fixed and optional fields
209        parts = line.split("\t")
210        status, subdir, testname = parts[0:3]
211        reason = parts[-1]
212        optional_parts = parts[3:-1]
213
214        # all the optional parts should be of the form "key=value"
215        assert sum('=' not in part for part in optional_parts) == 0
216        optional_fields = dict(part.split("=", 1)
217                               for part in optional_parts)
218
219        # build up a new status_line and return it
220        return cls(indent, status, subdir, testname, reason,
221                   optional_fields)
222
223
224class parser(base.parser):
225    @staticmethod
226    def make_job(dir):
227        return job(dir)
228
229
230    def state_iterator(self, buffer):
231        new_tests = []
232        boot_count = 0
233        group_subdir = None
234        sought_level = 0
235        stack = status_lib.status_stack()
236        current_kernel = kernel(self.job)
237        boot_in_progress = False
238        alert_pending = None
239        started_time = None
240
241        while not self.finished or buffer.size():
242            # stop processing once the buffer is empty
243            if buffer.size() == 0:
244                yield new_tests
245                new_tests = []
246                continue
247
248            # parse the next line
249            line = buffer.get()
250            tko_utils.dprint('\nSTATUS: ' + line.strip())
251            line = status_line.parse_line(line)
252            if line is None:
253                tko_utils.dprint('non-status line, ignoring')
254                continue # ignore non-status lines
255
256            # have we hit the job start line?
257            if (line.type == "START" and not line.subdir and
258                not line.testname):
259                sought_level = 1
260                tko_utils.dprint("found job level start "
261                                 "marker, looking for level "
262                                 "1 groups now")
263                continue
264
265            # have we hit the job end line?
266            if (line.type == "END" and not line.subdir and
267                not line.testname):
268                tko_utils.dprint("found job level end "
269                                 "marker, looking for level "
270                                 "0 lines now")
271                sought_level = 0
272
273            # START line, just push another layer on to the stack
274            # and grab the start time if this is at the job level
275            # we're currently seeking
276            if line.type == "START":
277                group_subdir = None
278                stack.start()
279                if line.indent == sought_level:
280                    started_time = \
281                                 tko_utils.get_timestamp(
282                        line.optional_fields, "timestamp")
283                tko_utils.dprint("start line, ignoring")
284                continue
285            # otherwise, update the status on the stack
286            else:
287                tko_utils.dprint("GROPE_STATUS: %s" %
288                                 [stack.current_status(),
289                                  line.status, line.subdir,
290                                  line.testname, line.reason])
291                stack.update(line.status)
292
293            if line.status == "ALERT":
294                tko_utils.dprint("job level alert, recording")
295                alert_pending = line.reason
296                continue
297
298            # ignore Autotest.install => GOOD lines
299            if (line.testname == "Autotest.install" and
300                line.status == "GOOD"):
301                tko_utils.dprint("Successful Autotest "
302                                 "install, ignoring")
303                continue
304
305            # ignore END lines for a reboot group
306            if (line.testname == "reboot" and line.type == "END"):
307                tko_utils.dprint("reboot group, ignoring")
308                continue
309
310            # convert job-level ABORTs into a 'JOB' test, and
311            # ignore other job-level events
312            if line.testname is None:
313                if (line.status == "ABORT" and
314                    line.type != "END"):
315                    line.testname = "JOB"
316                else:
317                    tko_utils.dprint("job level event, "
318                                    "ignoring")
319                    continue
320
321            # use the group subdir for END lines
322            if line.type == "END":
323                line.subdir = group_subdir
324
325            # are we inside a block group?
326            if (line.indent != sought_level and
327                line.status != "ABORT" and
328                not line.testname.startswith('reboot.')):
329                if line.subdir:
330                    tko_utils.dprint("set group_subdir: "
331                                     + line.subdir)
332                    group_subdir = line.subdir
333                tko_utils.dprint("ignoring incorrect indent "
334                                 "level %d != %d," %
335                                 (line.indent, sought_level))
336                continue
337
338            # use the subdir as the testname, except for
339            # boot.* and kernel.* tests
340            if (line.testname is None or
341                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
342                              line.testname)):
343                if line.subdir and '.' in line.subdir:
344                    line.testname = line.subdir
345
346            # has a reboot started?
347            if line.testname == "reboot.start":
348                started_time = tko_utils.get_timestamp(
349                    line.optional_fields, "timestamp")
350                tko_utils.dprint("reboot start event, "
351                                 "ignoring")
352                boot_in_progress = True
353                continue
354
355            # has a reboot finished?
356            if line.testname == "reboot.verify":
357                line.testname = "boot.%d" % boot_count
358                tko_utils.dprint("reboot verified")
359                boot_in_progress = False
360                verify_ident = line.reason.strip()
361                current_kernel = kernel(self.job, verify_ident)
362                boot_count += 1
363
364            if alert_pending:
365                line.status = "ALERT"
366                line.reason = alert_pending
367                alert_pending = None
368
369            # create the actual test object
370            finished_time = tko_utils.get_timestamp(
371                line.optional_fields, "timestamp")
372            final_status = stack.end()
373            tko_utils.dprint("Adding: "
374                             "%s\nSubdir:%s\nTestname:%s\n%s" %
375                             (final_status, line.subdir,
376                              line.testname, line.reason))
377            new_test = test.parse_test(self.job, line.subdir,
378                                       line.testname,
379                                       final_status, line.reason,
380                                       current_kernel,
381                                       started_time,
382                                       finished_time)
383            started_time = None
384            new_tests.append(new_test)
385
386        # the job is finished, but we never came back from reboot
387        if boot_in_progress:
388            testname = "boot.%d" % boot_count
389            reason = "machine did not return from reboot"
390            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
391                              "Testname:%s\n%s")
392                             % (testname, reason))
393            new_test = test.parse_test(self.job, None, testname,
394                                       "ABORT", reason,
395                                       current_kernel, None, None)
396            new_tests.append(new_test)
397        yield new_tests
398