version_0.py revision 5350a34676f05c45fdbf3c2b9ba234b7c6ed292a
1import re, os 2 3from autotest_lib.client.common_lib import utils as common_utils 4from autotest_lib.tko import utils as tko_utils, models, status_lib 5from autotest_lib.tko.parsers import base 6 7 8class NoHostnameError(Exception): 9 pass 10 11 12class job(models.job): 13 def __init__(self, dir): 14 job_dict = job.load_from_dir(dir) 15 super(job, self).__init__(dir, **job_dict) 16 17 18 @classmethod 19 def load_from_dir(cls, dir): 20 keyval = cls.read_keyval(dir) 21 tko_utils.dprint(str(keyval)) 22 23 user = keyval.get("user", None) 24 label = keyval.get("label", None) 25 host_group_name = keyval.get("host_group_name", None) 26 machine = keyval.get("hostname", None) 27 if not host_group_name and machine and "," in machine: 28 try: 29 machine = job.find_hostname(dir) # find a unique hostname 30 except NoHostnameError: 31 pass # just use the comma-separated name 32 queued_time = tko_utils.get_timestamp(keyval, "job_queued") 33 started_time = tko_utils.get_timestamp(keyval, "job_started") 34 finished_time = tko_utils.get_timestamp(keyval, "job_finished") 35 machine_owner = keyval.get("owner", None) 36 37 aborted_by = keyval.get("aborted_by", None) 38 aborted_at = tko_utils.get_timestamp(keyval, "aborted_on") 39 40 tko_utils.dprint("MACHINE NAME: %s" % machine) 41 if host_group_name and ((machine and "," in machine) or not machine): 42 tko_utils.dprint("Using host_group_name %r instead of " 43 "machine name." % host_group_name) 44 machine = host_group_name 45 46 return {"user": user, "label": label, "machine": machine, 47 "queued_time": queued_time, "started_time": started_time, 48 "finished_time": finished_time, "machine_owner": machine_owner, 49 "aborted_by": aborted_by, "aborted_on": aborted_at} 50 51 52 @staticmethod 53 def find_hostname(path): 54 hostname = os.path.join(path, "sysinfo", "hostname") 55 try: 56 machine = open(hostname).readline().rstrip() 57 return machine 58 except Exception: 59 tko_utils.dprint("Could not read a hostname from " 60 "sysinfo/hostname") 61 62 uname = os.path.join(path, "sysinfo", "uname_-a") 63 try: 64 machine = open(uname).readline().split()[1] 65 return machine 66 except Exception: 67 tko_utils.dprint("Could not read a hostname from " 68 "sysinfo/uname_-a") 69 70 raise NoHostnameError("Unable to find a machine name") 71 72 73class kernel(models.kernel): 74 def __init__(self, job, verify_ident=None): 75 kernel_dict = kernel.load_from_dir(job.dir, verify_ident) 76 super(kernel, self).__init__(**kernel_dict) 77 78 79 @staticmethod 80 def load_from_dir(dir, verify_ident=None): 81 # try and load the booted kernel version 82 attributes = False 83 i = 1 84 build_dir = os.path.join(dir, "build") 85 while True: 86 if not os.path.exists(build_dir): 87 break 88 build_log = os.path.join(build_dir, "debug", "build_log") 89 attributes = kernel.load_from_build_log(build_log) 90 if attributes: 91 break 92 i += 1 93 build_dir = os.path.join(dir, "build.%d" % (i)) 94 95 if not attributes: 96 if verify_ident: 97 base = verify_ident 98 else: 99 base = kernel.load_from_sysinfo(dir) 100 patches = [] 101 hashes = [] 102 else: 103 base, patches, hashes = attributes 104 tko_utils.dprint("kernel.__init__() found kernel version %s" 105 % base) 106 107 # compute the kernel hash 108 if base == "UNKNOWN": 109 kernel_hash = "UNKNOWN" 110 else: 111 kernel_hash = kernel.compute_hash(base, hashes) 112 113 return {"base": base, "patches": patches, 114 "kernel_hash": kernel_hash} 115 116 117 @staticmethod 118 def load_from_sysinfo(path): 119 for subdir in ("reboot1", ""): 120 uname_path = os.path.join(path, "sysinfo", subdir, 121 "uname_-a") 122 if not os.path.exists(uname_path): 123 continue 124 uname = open(uname_path).readline().split() 125 return re.sub("-autotest$", "", uname[2]) 126 return "UNKNOWN" 127 128 129 @staticmethod 130 def load_from_build_log(path): 131 if not os.path.exists(path): 132 return None 133 134 base, patches, hashes = "UNKNOWN", [], [] 135 for line in file(path): 136 head, rest = line.split(": ", 1) 137 rest = rest.split() 138 if head == "BASE": 139 base = rest[0] 140 elif head == "PATCH": 141 patches.append(patch(*rest)) 142 hashes.append(rest[2]) 143 return base, patches, hashes 144 145 146class test(models.test): 147 def __init__(self, subdir, testname, status, reason, test_kernel, 148 machine, started_time, finished_time, iterations, 149 attributes): 150 # for backwards compatibility with the original parser 151 # implementation, if there is no test version we need a NULL 152 # value to be used; also, if there is a version it should 153 # be terminated by a newline 154 if "version" in attributes: 155 attributes["version"] = str(attributes["version"]) 156 else: 157 attributes["version"] = None 158 159 super(test, self).__init__(subdir, testname, status, reason, 160 test_kernel, machine, started_time, 161 finished_time, iterations, 162 attributes) 163 164 165 @staticmethod 166 def load_iterations(keyval_path): 167 return iteration.load_from_keyval(keyval_path) 168 169 170class patch(models.patch): 171 def __init__(self, spec, reference, hash): 172 tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash)) 173 super(patch, self).__init__(spec, reference, hash) 174 self.spec = spec 175 self.reference = reference 176 self.hash = hash 177 178 179class iteration(models.iteration): 180 @staticmethod 181 def parse_line_into_dicts(line, attr_dict, perf_dict): 182 key, value = line.split("=", 1) 183 perf_dict[key] = value 184 185 186class status_line(object): 187 def __init__(self, indent, status, subdir, testname, reason, 188 optional_fields): 189 # pull out the type & status of the line 190 if status == "START": 191 self.type = "START" 192 self.status = None 193 elif status.startswith("END "): 194 self.type = "END" 195 self.status = status[4:] 196 else: 197 self.type = "STATUS" 198 self.status = status 199 assert (self.status is None or 200 self.status in status_lib.statuses) 201 202 # save all the other parameters 203 self.indent = indent 204 self.subdir = self.parse_name(subdir) 205 self.testname = self.parse_name(testname) 206 self.reason = reason 207 self.optional_fields = optional_fields 208 209 210 @staticmethod 211 def parse_name(name): 212 if name == "----": 213 return None 214 return name 215 216 217 @staticmethod 218 def is_status_line(line): 219 return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None 220 221 222 @classmethod 223 def parse_line(cls, line): 224 if not status_line.is_status_line(line): 225 return None 226 indent, line = re.search(r"^(\t*)(.*)$", line).groups() 227 indent = len(indent) 228 229 # split the line into the fixed and optional fields 230 parts = line.split("\t") 231 status, subdir, testname = parts[0:3] 232 reason = parts[-1] 233 optional_parts = parts[3:-1] 234 235 # all the optional parts should be of the form "key=value" 236 assert sum('=' not in part for part in optional_parts) == 0 237 optional_fields = dict(part.split("=", 1) 238 for part in optional_parts) 239 240 # build up a new status_line and return it 241 return cls(indent, status, subdir, testname, reason, 242 optional_fields) 243 244 245class parser(base.parser): 246 @staticmethod 247 def make_job(dir): 248 return job(dir) 249 250 251 def state_iterator(self, buffer): 252 new_tests = [] 253 boot_count = 0 254 group_subdir = None 255 sought_level = 0 256 stack = status_lib.status_stack() 257 current_kernel = kernel(self.job) 258 boot_in_progress = False 259 alert_pending = None 260 started_time = None 261 262 while not self.finished or buffer.size(): 263 # stop processing once the buffer is empty 264 if buffer.size() == 0: 265 yield new_tests 266 new_tests = [] 267 continue 268 269 # parse the next line 270 line = buffer.get() 271 tko_utils.dprint('\nSTATUS: ' + line.strip()) 272 line = status_line.parse_line(line) 273 if line is None: 274 tko_utils.dprint('non-status line, ignoring') 275 continue # ignore non-status lines 276 277 # have we hit the job start line? 278 if (line.type == "START" and not line.subdir and 279 not line.testname): 280 sought_level = 1 281 tko_utils.dprint("found job level start " 282 "marker, looking for level " 283 "1 groups now") 284 continue 285 286 # have we hit the job end line? 287 if (line.type == "END" and not line.subdir and 288 not line.testname): 289 tko_utils.dprint("found job level end " 290 "marker, looking for level " 291 "0 lines now") 292 sought_level = 0 293 294 # START line, just push another layer on to the stack 295 # and grab the start time if this is at the job level 296 # we're currently seeking 297 if line.type == "START": 298 group_subdir = None 299 stack.start() 300 if line.indent == sought_level: 301 started_time = \ 302 tko_utils.get_timestamp( 303 line.optional_fields, "timestamp") 304 tko_utils.dprint("start line, ignoring") 305 continue 306 # otherwise, update the status on the stack 307 else: 308 tko_utils.dprint("GROPE_STATUS: %s" % 309 [stack.current_status(), 310 line.status, line.subdir, 311 line.testname, line.reason]) 312 stack.update(line.status) 313 314 if line.status == "ALERT": 315 tko_utils.dprint("job level alert, recording") 316 alert_pending = line.reason 317 continue 318 319 # ignore Autotest.install => GOOD lines 320 if (line.testname == "Autotest.install" and 321 line.status == "GOOD"): 322 tko_utils.dprint("Successful Autotest " 323 "install, ignoring") 324 continue 325 326 # ignore END lines for a reboot group 327 if (line.testname == "reboot" and line.type == "END"): 328 tko_utils.dprint("reboot group, ignoring") 329 continue 330 331 # convert job-level ABORTs into a 'CLIENT_JOB' test, and 332 # ignore other job-level events 333 if line.testname is None: 334 if (line.status == "ABORT" and 335 line.type != "END"): 336 line.testname = "CLIENT_JOB" 337 else: 338 tko_utils.dprint("job level event, " 339 "ignoring") 340 continue 341 342 # use the group subdir for END lines 343 if line.type == "END": 344 line.subdir = group_subdir 345 346 # are we inside a block group? 347 if (line.indent != sought_level and 348 line.status != "ABORT" and 349 not line.testname.startswith('reboot.')): 350 if line.subdir: 351 tko_utils.dprint("set group_subdir: " 352 + line.subdir) 353 group_subdir = line.subdir 354 tko_utils.dprint("ignoring incorrect indent " 355 "level %d != %d," % 356 (line.indent, sought_level)) 357 continue 358 359 # use the subdir as the testname, except for 360 # boot.* and kernel.* tests 361 if (line.testname is None or 362 not re.search(r"^(boot(\.\d+)?$|kernel\.)", 363 line.testname)): 364 if line.subdir and '.' in line.subdir: 365 line.testname = line.subdir 366 367 # has a reboot started? 368 if line.testname == "reboot.start": 369 started_time = tko_utils.get_timestamp( 370 line.optional_fields, "timestamp") 371 tko_utils.dprint("reboot start event, " 372 "ignoring") 373 boot_in_progress = True 374 continue 375 376 # has a reboot finished? 377 if line.testname == "reboot.verify": 378 line.testname = "boot.%d" % boot_count 379 tko_utils.dprint("reboot verified") 380 boot_in_progress = False 381 verify_ident = line.reason.strip() 382 current_kernel = kernel(self.job, verify_ident) 383 boot_count += 1 384 385 if alert_pending: 386 line.status = "ALERT" 387 line.reason = alert_pending 388 alert_pending = None 389 390 # create the actual test object 391 finished_time = tko_utils.get_timestamp( 392 line.optional_fields, "timestamp") 393 final_status = stack.end() 394 tko_utils.dprint("Adding: " 395 "%s\nSubdir:%s\nTestname:%s\n%s" % 396 (final_status, line.subdir, 397 line.testname, line.reason)) 398 new_test = test.parse_test(self.job, line.subdir, 399 line.testname, 400 final_status, line.reason, 401 current_kernel, 402 started_time, 403 finished_time) 404 started_time = None 405 new_tests.append(new_test) 406 407 # the job is finished, but we never came back from reboot 408 if boot_in_progress: 409 testname = "boot.%d" % boot_count 410 reason = "machine did not return from reboot" 411 tko_utils.dprint(("Adding: ABORT\nSubdir:----\n" 412 "Testname:%s\n%s") 413 % (testname, reason)) 414 new_test = test.parse_test(self.job, None, testname, 415 "ABORT", reason, 416 current_kernel, None, None) 417 new_tests.append(new_test) 418 yield new_tests 419