base_sysinfo.py revision 9e33f275b86d24aabc7f09586f62f94a1d876c6f
1import os, shutil, re, glob, subprocess 2 3from autotest_lib.client.common_lib import log 4from autotest_lib.client.bin import utils, package 5 6 7_DEFAULT_COMMANDS_TO_LOG_PER_TEST = [] 8_DEFAULT_COMMANDS_TO_LOG_PER_BOOT = [ 9 "lspci -vvn", "gcc --version", "ld --version", "mount", "hostname", 10 "uptime", 11 ] 12_DEFAULT_COMMANDS_TO_LOG_BEFORE_ITERATION = [] 13_DEFAULT_COMMANDS_TO_LOG_AFTER_ITERATION = [] 14 15_DEFAULT_FILES_TO_LOG_PER_TEST = [] 16_DEFAULT_FILES_TO_LOG_PER_BOOT = [ 17 "/proc/pci", "/proc/meminfo", "/proc/slabinfo", "/proc/version", 18 "/proc/cpuinfo", "/proc/modules", "/proc/interrupts", 19 ] 20_DEFAULT_FILES_TO_LOG_BEFORE_ITERATION = [ 21 "/proc/schedstat" 22 ] 23_DEFAULT_FILES_TO_LOG_AFTER_ITERATION = [ 24 "/proc/schedstat" 25 ] 26 27 28class loggable(object): 29 """ Abstract class for representing all things "loggable" by sysinfo. """ 30 def __init__(self, logf, log_in_keyval): 31 self.logf = logf 32 self.log_in_keyval = log_in_keyval 33 34 35 def readline(self, logdir): 36 path = os.path.join(logdir, self.logf) 37 if os.path.exists(path): 38 return utils.read_one_line(path) 39 else: 40 return "" 41 42 43class logfile(loggable): 44 def __init__(self, path, logf=None, log_in_keyval=False): 45 if not logf: 46 logf = os.path.basename(path) 47 super(logfile, self).__init__(logf, log_in_keyval) 48 self.path = path 49 50 51 def __repr__(self): 52 r = "sysinfo.logfile(%r, %r, %r)" 53 r %= (self.path, self.logf, self.log_in_keyval) 54 return r 55 56 57 def __eq__(self, other): 58 if isinstance(other, logfile): 59 return (self.path, self.logf) == (other.path, other.logf) 60 elif isinstance(other, loggable): 61 return False 62 return NotImplemented 63 64 65 def __ne__(self, other): 66 result = self.__eq__(other) 67 if result is NotImplemented: 68 return result 69 return not result 70 71 72 def __hash__(self): 73 return hash((self.path, self.logf)) 74 75 76 def run(self, logdir): 77 if os.path.exists(self.path): 78 shutil.copy(self.path, os.path.join(logdir, self.logf)) 79 80 81class command(loggable): 82 def __init__(self, cmd, logf=None, log_in_keyval=False): 83 if not logf: 84 logf = cmd.replace(" ", "_") 85 super(command, self).__init__(logf, log_in_keyval) 86 self.cmd = cmd 87 88 89 def __repr__(self): 90 r = "sysinfo.command(%r, %r, %r)" 91 r %= (self.cmd, self.logf, self.log_in_keyval) 92 return r 93 94 95 def __eq__(self, other): 96 if isinstance(other, command): 97 return (self.cmd, self.logf) == (other.cmd, other.logf) 98 elif isinstance(other, loggable): 99 return False 100 return NotImplemented 101 102 103 def __ne__(self, other): 104 result = self.__eq__(other) 105 if result is NotImplemented: 106 return result 107 return not result 108 109 110 def __hash__(self): 111 return hash((self.cmd, self.logf)) 112 113 114 def run(self, logdir): 115 stdin = open(os.devnull, "r") 116 stdout = open(os.path.join(logdir, self.logf), "w") 117 stderr = open(os.devnull, "w") 118 env = os.environ.copy() 119 if "PATH" not in env: 120 env["PATH"] = "/usr/bin:/bin" 121 subprocess.call(self.cmd, stdin=stdin, stdout=stdout, stderr=stderr, 122 shell=True, env=env) 123 for f in (stdin, stdout, stderr): 124 f.close() 125 126 127class base_sysinfo(object): 128 def __init__(self, job_resultsdir): 129 self.sysinfodir = self._get_sysinfodir(job_resultsdir) 130 131 # pull in the post-test logs to collect 132 self.test_loggables = set() 133 for cmd in _DEFAULT_COMMANDS_TO_LOG_PER_TEST: 134 self.test_loggables.add(command(cmd)) 135 for filename in _DEFAULT_FILES_TO_LOG_PER_TEST: 136 self.test_loggables.add(logfile(filename)) 137 138 # pull in the EXTRA post-boot logs to collect 139 self.boot_loggables = set() 140 for cmd in _DEFAULT_COMMANDS_TO_LOG_PER_BOOT: 141 self.boot_loggables.add(command(cmd)) 142 for filename in _DEFAULT_FILES_TO_LOG_PER_BOOT: 143 self.boot_loggables.add(logfile(filename)) 144 145 # pull in the pre test iteration logs to collect 146 self.before_iteration_loggables = set() 147 for cmd in _DEFAULT_COMMANDS_TO_LOG_BEFORE_ITERATION: 148 self.before_iteration_loggables.add( 149 command(cmd, logf=cmd.replace(" ", "_") + '.before')) 150 for fname in _DEFAULT_FILES_TO_LOG_BEFORE_ITERATION: 151 self.before_iteration_loggables.add( 152 logfile(fname, logf=os.path.basename(fname) + '.before')) 153 154 # pull in the post test iteration logs to collect 155 self.after_iteration_loggables = set() 156 for cmd in _DEFAULT_COMMANDS_TO_LOG_AFTER_ITERATION: 157 self.after_iteration_loggables.add( 158 command(cmd, logf=cmd.replace(" ", "_") + '.after')) 159 for fname in _DEFAULT_FILES_TO_LOG_AFTER_ITERATION: 160 self.after_iteration_loggables.add( 161 logfile(fname, logf=os.path.basename(fname) + '.after')) 162 163 # add in a couple of extra files and commands we want to grab 164 self.test_loggables.add(command("df -mP", logf="df")) 165 self.test_loggables.add(command("dmesg -c", logf="dmesg")) 166 self.boot_loggables.add(logfile("/proc/cmdline", 167 log_in_keyval=True)) 168 self.boot_loggables.add(command("uname -a", logf="uname", 169 log_in_keyval=True)) 170 171 172 def serialize(self): 173 return {"boot": self.boot_loggables, "test": self.test_loggables} 174 175 176 def deserialize(self, serialized): 177 self.boot_loggables = serialized["boot"] 178 self.test_loggables = serialized["test"] 179 180 181 @staticmethod 182 def _get_sysinfodir(resultsdir): 183 sysinfodir = os.path.join(resultsdir, "sysinfo") 184 if not os.path.exists(sysinfodir): 185 os.makedirs(sysinfodir) 186 return sysinfodir 187 188 189 def _get_reboot_count(self): 190 if not glob.glob(os.path.join(self.sysinfodir, "*")): 191 return -1 192 else: 193 return len(glob.glob(os.path.join(self.sysinfodir, "boot.*"))) 194 195 196 def _get_boot_subdir(self, next=False): 197 reboot_count = self._get_reboot_count() 198 if next: 199 reboot_count += 1 200 if reboot_count < 1: 201 return self.sysinfodir 202 else: 203 boot_dir = "boot.%d" % (reboot_count - 1) 204 return os.path.join(self.sysinfodir, boot_dir) 205 206 207 def _get_iteration_subdir(self, iteration): 208 iter_dir = "iteration.%d" % iteration 209 210 logdir = os.path.join(self.sysinfodir, iter_dir) 211 if not os.path.exists(logdir): 212 os.mkdir(logdir) 213 return logdir 214 215 216 @log.log_and_ignore_errors("post-reboot sysinfo error:") 217 def log_per_reboot_data(self): 218 """ Logging hook called whenever a job starts, and again after 219 any reboot. """ 220 logdir = self._get_boot_subdir(next=True) 221 if not os.path.exists(logdir): 222 os.mkdir(logdir) 223 224 for log in (self.test_loggables | self.boot_loggables): 225 log.run(logdir) 226 227 # also log any installed packages 228 installed_path = os.path.join(logdir, "installed_packages") 229 installed_packages = "\n".join(package.list_all()) + "\n" 230 utils.open_write_close(installed_path, installed_packages) 231 232 233 @log.log_and_ignore_errors("pre-test sysinfo error:") 234 def log_before_each_test(self, test): 235 """ Logging hook called before a test starts. """ 236 self._installed_packages = package.list_all() 237 if os.path.exists("/var/log/messages"): 238 stat = os.stat("/var/log/messages") 239 self._messages_size = stat.st_size 240 self._messages_inode = stat.st_ino 241 242 243 @log.log_and_ignore_errors("post-test sysinfo error:") 244 def log_after_each_test(self, test): 245 """ Logging hook called after a test finishs. """ 246 test_sysinfodir = self._get_sysinfodir(test.outputdir) 247 248 # create a symlink in the test sysinfo dir to the current boot 249 reboot_dir = self._get_boot_subdir() 250 assert os.path.exists(reboot_dir) 251 symlink_dest = os.path.join(test_sysinfodir, "reboot_current") 252 os.symlink(reboot_dir, symlink_dest) 253 254 # run all the standard logging commands 255 for log in self.test_loggables: 256 log.run(test_sysinfodir) 257 258 # grab any new data from /var/log/messages 259 self._log_messages(test_sysinfodir) 260 261 # log some sysinfo data into the test keyval file 262 keyval = self.log_test_keyvals(test_sysinfodir) 263 test.write_test_keyval(keyval) 264 265 # log any changes to installed packages 266 old_packages = set(self._installed_packages) 267 new_packages = set(package.list_all()) 268 added_path = os.path.join(test_sysinfodir, "added_packages") 269 added_packages = "\n".join(new_packages - old_packages) + "\n" 270 utils.open_write_close(added_path, added_packages) 271 removed_path = os.path.join(test_sysinfodir, "removed_packages") 272 removed_packages = "\n".join(old_packages - new_packages) + "\n" 273 utils.open_write_close(removed_path, removed_packages) 274 275 276 @log.log_and_ignore_errors("pre-test siteration sysinfo error:") 277 def log_before_each_iteration(self, test, iteration=None): 278 """ Logging hook called before a test iteration.""" 279 if not iteration: 280 iteration = test.iteration 281 logdir = self._get_iteration_subdir(iteration) 282 283 for log in self.before_iteration_loggables: 284 log.run(logdir) 285 286 287 @log.log_and_ignore_errors("post-test siteration sysinfo error:") 288 def log_after_each_iteration(self, test, iteration=None): 289 """ Logging hook called after a test iteration.""" 290 if not iteration: 291 iteration = test.iteration 292 logdir = self._get_iteration_subdir(iteration) 293 294 for log in self.after_iteration_loggables: 295 log.run(logdir) 296 297 298 def _log_messages(self, logdir): 299 """ Log all of the new data in /var/log/messages. """ 300 try: 301 # log all of the new data in /var/log/messages 302 bytes_to_skip = 0 303 if hasattr(self, "_messages_size"): 304 current_inode = os.stat("/var/log/messages").st_ino 305 if current_inode == self._messages_inode: 306 bytes_to_skip = self._messages_size 307 in_messages = open("/var/log/messages") 308 in_messages.seek(bytes_to_skip) 309 out_messages = open(os.path.join(logdir, "messages"), "w") 310 out_messages.write(in_messages.read()) 311 in_messages.close() 312 out_messages.close() 313 except Exception, e: 314 print "/var/log/messages collection failed with %s" % e 315 316 317 @staticmethod 318 def _read_sysinfo_keyvals(loggables, logdir): 319 keyval = {} 320 for log in loggables: 321 if log.log_in_keyval: 322 keyval["sysinfo-" + log.logf] = log.readline(logdir) 323 return keyval 324 325 326 def log_test_keyvals(self, test_sysinfodir): 327 """ Logging hook called by log_after_each_test to collect keyval 328 entries to be written in the test keyval. """ 329 keyval = {} 330 331 # grab any loggables that should be in the keyval 332 keyval.update(self._read_sysinfo_keyvals( 333 self.test_loggables, test_sysinfodir)) 334 keyval.update(self._read_sysinfo_keyvals( 335 self.boot_loggables, 336 os.path.join(test_sysinfodir, "reboot_current"))) 337 338 # remove hostname from uname info 339 # Linux lpt36 2.6.18-smp-230.1 #1 [4069269] SMP Fri Oct 24 11:30:... 340 if "sysinfo-uname" in keyval: 341 kernel_vers = " ".join(keyval["sysinfo-uname"].split()[2:]) 342 keyval["sysinfo-uname"] = kernel_vers 343 344 # grab the total avail memory, not used by sys tables 345 path = os.path.join(test_sysinfodir, "reboot_current", "meminfo") 346 if os.path.exists(path): 347 mem_data = open(path).read() 348 match = re.search(r"^MemTotal:\s+(\d+) kB$", mem_data, 349 re.MULTILINE) 350 if match: 351 keyval["sysinfo-memtotal-in-kb"] = match.group(1) 352 353 # guess the system's total physical memory, including sys tables 354 keyval["sysinfo-phys-mbytes"] = utils.rounded_memtotal()//1024 355 356 # return what we collected 357 return keyval 358