version_0.py revision f1ae354808a2eeb95d706a669250b613765212a4 
1import re, os
2
3from autotest_lib.client.common_lib import utils as common_utils
4from autotest_lib.tko import utils as tko_utils, models, status_lib
5from autotest_lib.tko.parsers import base
6
7
8class NoHostnameError(Exception):
9    pass
10
11
12class job(models.job):
13    def __init__(self, dir):
14        job_dict = job.load_from_dir(dir)
15        super(job, self).__init__(dir, **job_dict)
16
17
18    @classmethod
19    def load_from_dir(cls, dir):
20        keyval = cls.read_keyval(dir)
21        tko_utils.dprint(str(keyval))
22
23        user = keyval.get("user", None)
24        label = keyval.get("label", None)
25        host_group_name = keyval.get("host_group_name", None)
26        machine = keyval.get("hostname", None)
27        if machine and "," in machine:
28            try:
29                machine = job.find_hostname(dir) # find a unique hostname
30            except NoHostnameError:
31                pass  # just use the comma-separated name
32        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
33        started_time = tko_utils.get_timestamp(keyval, "job_started")
34        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
35        machine_owner = keyval.get("owner", None)
36
37        aborted_by = keyval.get("aborted_by", None)
38        aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
39
40        tko_utils.dprint("MACHINE NAME: %s" % machine)
41        if host_group_name and ((machine and "," in machine) or not machine):
42            tko_utils.dprint("Using host_group_name %r instead of "
43                             "machine name." % host_group_name)
44            machine = host_group_name
45
46        return {"user": user, "label": label, "machine": machine,
47                "queued_time": queued_time, "started_time": started_time,
48                "finished_time": finished_time, "machine_owner": machine_owner,
49                "aborted_by": aborted_by, "aborted_on": aborted_at}
50
51
52    @staticmethod
53    def find_hostname(path):
54        hostname = os.path.join(path, "sysinfo", "hostname")
55        try:
56            machine = open(hostname).readline().rstrip()
57            return machine
58        except Exception:
59            tko_utils.dprint("Could not read a hostname from "
60                             "sysinfo/hostname")
61
62        uname = os.path.join(path, "sysinfo", "uname_-a")
63        try:
64            machine = open(uname).readline().split()[1]
65            return machine
66        except Exception:
67            tko_utils.dprint("Could not read a hostname from "
68                             "sysinfo/uname_-a")
69
70        raise NoHostnameError("Unable to find a machine name")
71
72
73class kernel(models.kernel):
74    def __init__(self, job, verify_ident=None):
75        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
76        super(kernel, self).__init__(**kernel_dict)
77
78
79    @staticmethod
80    def load_from_dir(dir, verify_ident=None):
81        # try and load the booted kernel version
82	attributes = False
83	i = 1
84	build_dir = os.path.join(dir, "build")
85	while True:
86	    if not os.path.exists(build_dir):
87		break
88	    build_log = os.path.join(build_dir, "debug", "build_log")
89            attributes = kernel.load_from_build_log(build_log)
90	    if attributes:
91		break
92	    i += 1
93	    build_dir = os.path.join(dir, "build.%d" % (i))
94
95        if not attributes:
96            if verify_ident:
97                base = verify_ident
98            else:
99                base = kernel.load_from_sysinfo(dir)
100            patches = []
101            hashes = []
102        else:
103            base, patches, hashes = attributes
104        tko_utils.dprint("kernel.__init__() found kernel version %s"
105                         % base)
106
107        # compute the kernel hash
108        if base == "UNKNOWN":
109            kernel_hash = "UNKNOWN"
110        else:
111            kernel_hash = kernel.compute_hash(base, hashes)
112
113        return {"base": base, "patches": patches,
114                "kernel_hash": kernel_hash}
115
116
117    @staticmethod
118    def load_from_sysinfo(path):
119        for subdir in ("reboot1", ""):
120            uname_path = os.path.join(path, "sysinfo", subdir,
121                                      "uname_-a")
122            if not os.path.exists(uname_path):
123                continue
124            uname = open(uname_path).readline().split()
125            return re.sub("-autotest$", "", uname[2])
126        return "UNKNOWN"
127
128
129    @staticmethod
130    def load_from_build_log(path):
131        if not os.path.exists(path):
132            return None
133
134        base, patches, hashes = "UNKNOWN", [], []
135        for line in file(path):
136            head, rest = line.split(": ", 1)
137            rest = rest.split()
138            if head == "BASE":
139                base = rest[0]
140            elif head == "PATCH":
141                patches.append(patch(*rest))
142                hashes.append(rest[2])
143        return base, patches, hashes
144
145
146class test(models.test):
147    def __init__(self, subdir, testname, status, reason, test_kernel,
148                 machine, started_time, finished_time, iterations,
149                 attributes):
150        # for backwards compatibility with the original parser
151        # implementation, if there is no test version we need a NULL
152        # value to be used; also, if there is a version it should
153        # be terminated by a newline
154        if "version" in attributes:
155            attributes["version"] = str(attributes["version"])
156        else:
157            attributes["version"] = None
158
159        super(test, self).__init__(subdir, testname, status, reason,
160                                   test_kernel, machine, started_time,
161                                   finished_time, iterations,
162                                   attributes)
163
164
165    @staticmethod
166    def load_iterations(keyval_path):
167        return iteration.load_from_keyval(keyval_path)
168
169
170class patch(models.patch):
171    def __init__(self, spec, reference, hash):
172        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
173        super(patch, self).__init__(spec, reference, hash)
174        self.spec = spec
175        self.reference = reference
176        self.hash = hash
177
178
179class iteration(models.iteration):
180    @staticmethod
181    def parse_line_into_dicts(line, attr_dict, perf_dict):
182        key, value = line.split("=", 1)
183        perf_dict[key] = value
184
185
186class status_line(object):
187    def __init__(self, indent, status, subdir, testname, reason,
188                 optional_fields):
189        # pull out the type & status of the line
190        if status == "START":
191            self.type = "START"
192            self.status = None
193        elif status.startswith("END "):
194            self.type = "END"
195            self.status = status[4:]
196        else:
197            self.type = "STATUS"
198            self.status = status
199        assert (self.status is None or
200                self.status in status_lib.statuses)
201
202        # save all the other parameters
203        self.indent = indent
204        self.subdir = self.parse_name(subdir)
205        self.testname = self.parse_name(testname)
206        self.reason = reason
207        self.optional_fields = optional_fields
208
209
210    @staticmethod
211    def parse_name(name):
212        if name == "----":
213            return None
214        return name
215
216
217    @staticmethod
218    def is_status_line(line):
219        return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
220
221
222    @classmethod
223    def parse_line(cls, line):
224        if not status_line.is_status_line(line):
225            return None
226        indent, line = re.search(r"^(\t*)(.*)$", line).groups()
227        indent = len(indent)
228
229        # split the line into the fixed and optional fields
230        parts = line.split("\t")
231        status, subdir, testname = parts[0:3]
232        reason = parts[-1]
233        optional_parts = parts[3:-1]
234
235        # all the optional parts should be of the form "key=value"
236        assert sum('=' not in part for part in optional_parts) == 0
237        optional_fields = dict(part.split("=", 1)
238                               for part in optional_parts)
239
240        # build up a new status_line and return it
241        return cls(indent, status, subdir, testname, reason,
242                   optional_fields)
243
244
245class parser(base.parser):
246    @staticmethod
247    def make_job(dir):
248        return job(dir)
249
250
251    def state_iterator(self, buffer):
252        new_tests = []
253        boot_count = 0
254        group_subdir = None
255        sought_level = 0
256        stack = status_lib.status_stack()
257        current_kernel = kernel(self.job)
258        boot_in_progress = False
259        alert_pending = None
260        started_time = None
261
262        while not self.finished or buffer.size():
263            # stop processing once the buffer is empty
264            if buffer.size() == 0:
265                yield new_tests
266                new_tests = []
267                continue
268
269            # parse the next line
270            line = buffer.get()
271            tko_utils.dprint('\nSTATUS: ' + line.strip())
272            line = status_line.parse_line(line)
273            if line is None:
274                tko_utils.dprint('non-status line, ignoring')
275                continue # ignore non-status lines
276
277            # have we hit the job start line?
278            if (line.type == "START" and not line.subdir and
279                not line.testname):
280                sought_level = 1
281                tko_utils.dprint("found job level start "
282                                 "marker, looking for level "
283                                 "1 groups now")
284                continue
285
286            # have we hit the job end line?
287            if (line.type == "END" and not line.subdir and
288                not line.testname):
289                tko_utils.dprint("found job level end "
290                                 "marker, looking for level "
291                                 "0 lines now")
292                sought_level = 0
293
294            # START line, just push another layer on to the stack
295            # and grab the start time if this is at the job level
296            # we're currently seeking
297            if line.type == "START":
298                group_subdir = None
299                stack.start()
300                if line.indent == sought_level:
301                    started_time = \
302                                 tko_utils.get_timestamp(
303                        line.optional_fields, "timestamp")
304                tko_utils.dprint("start line, ignoring")
305                continue
306            # otherwise, update the status on the stack
307            else:
308                tko_utils.dprint("GROPE_STATUS: %s" %
309                                 [stack.current_status(),
310                                  line.status, line.subdir,
311                                  line.testname, line.reason])
312                stack.update(line.status)
313
314            if line.status == "ALERT":
315                tko_utils.dprint("job level alert, recording")
316                alert_pending = line.reason
317                continue
318
319            # ignore Autotest.install => GOOD lines
320            if (line.testname == "Autotest.install" and
321                line.status == "GOOD"):
322                tko_utils.dprint("Successful Autotest "
323                                 "install, ignoring")
324                continue
325
326            # ignore END lines for a reboot group
327            if (line.testname == "reboot" and line.type == "END"):
328                tko_utils.dprint("reboot group, ignoring")
329                continue
330
331            # convert job-level ABORTs into a 'CLIENT_JOB' test, and
332            # ignore other job-level events
333            if line.testname is None:
334                if (line.status == "ABORT" and
335                    line.type != "END"):
336                    line.testname = "CLIENT_JOB"
337                else:
338                    tko_utils.dprint("job level event, "
339                                    "ignoring")
340                    continue
341
342            # use the group subdir for END lines
343            if line.type == "END":
344                line.subdir = group_subdir
345
346            # are we inside a block group?
347            if (line.indent != sought_level and
348                line.status != "ABORT" and
349                not line.testname.startswith('reboot.')):
350                if line.subdir:
351                    tko_utils.dprint("set group_subdir: "
352                                     + line.subdir)
353                    group_subdir = line.subdir
354                tko_utils.dprint("ignoring incorrect indent "
355                                 "level %d != %d," %
356                                 (line.indent, sought_level))
357                continue
358
359            # use the subdir as the testname, except for
360            # boot.* and kernel.* tests
361            if (line.testname is None or
362                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
363                              line.testname)):
364                if line.subdir and '.' in line.subdir:
365                    line.testname = line.subdir
366
367            # has a reboot started?
368            if line.testname == "reboot.start":
369                started_time = tko_utils.get_timestamp(
370                    line.optional_fields, "timestamp")
371                tko_utils.dprint("reboot start event, "
372                                 "ignoring")
373                boot_in_progress = True
374                continue
375
376            # has a reboot finished?
377            if line.testname == "reboot.verify":
378                line.testname = "boot.%d" % boot_count
379                tko_utils.dprint("reboot verified")
380                boot_in_progress = False
381                verify_ident = line.reason.strip()
382                current_kernel = kernel(self.job, verify_ident)
383                boot_count += 1
384
385            if alert_pending:
386                line.status = "ALERT"
387                line.reason = alert_pending
388                alert_pending = None
389
390            # create the actual test object
391            finished_time = tko_utils.get_timestamp(
392                line.optional_fields, "timestamp")
393            final_status = stack.end()
394            tko_utils.dprint("Adding: "
395                             "%s\nSubdir:%s\nTestname:%s\n%s" %
396                             (final_status, line.subdir,
397                              line.testname, line.reason))
398            new_test = test.parse_test(self.job, line.subdir,
399                                       line.testname,
400                                       final_status, line.reason,
401                                       current_kernel,
402                                       started_time,
403                                       finished_time)
404            started_time = None
405            new_tests.append(new_test)
406
407        # the job is finished, but we never came back from reboot
408        if boot_in_progress:
409            testname = "boot.%d" % boot_count
410            reason = "machine did not return from reboot"
411            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
412                              "Testname:%s\n%s")
413                             % (testname, reason))
414            new_test = test.parse_test(self.job, None, testname,
415                                       "ABORT", reason,
416                                       current_kernel, None, None)
417            new_tests.append(new_test)
418        yield new_tests
419