test.py revision e3a22513ce0a52739f89589c43af9e221e1ada61
14f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# Shell class for a test, inherited by all individual tests 2e83b0cadc67882c1ba7f430d16dab80c9b3a0228Dan Handley# 34f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# Methods: 44f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# __init__ initialise 54f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# initialize run once for each job 64f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# setup run once for each new version of the test installed 74f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# run run the test (wrapped by job.run_test()) 84f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# 94f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# Data: 104f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# job backreference to the job this test instance is part of 114f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# outputdir eg. results/<job>/<testname.tag> 124f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# resultsdir eg. results/<job>/<testname.tag>/results 134f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# profdir eg. results/<job>/<testname.tag>/profiling 144f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# debugdir eg. results/<job>/<testname.tag>/debug 154f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# bindir eg. tests/<test> 164f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# src eg. tests/<test>/src 174f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta# tmpdir eg. tmp/<tempname>_<testname.tag> 184f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 194f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Guptaimport fcntl, getpass, os, re, sys, shutil, tarfile, tempfile, time, traceback 204f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Guptaimport warnings, logging, glob, resource 214f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 224f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Guptafrom autotest_lib.client.common_lib import error 234f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Guptafrom autotest_lib.client.bin import utils 244f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 254f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 264f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Guptaclass base_test: 274f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta preserve_srcdir = False 284f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta network_destabilizing = False 294f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 304f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def __init__(self, job, bindir, outputdir): 314f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.job = job 324f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.pkgmgr = job.pkgmgr 33caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.autodir = job.autodir 34caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.outputdir = outputdir 35a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan self.tagged_testname = os.path.basename(self.outputdir) 36caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.resultsdir = os.path.join(self.outputdir, 'results') 374f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta os.mkdir(self.resultsdir) 384f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.profdir = os.path.join(self.outputdir, 'profiling') 39caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan os.mkdir(self.profdir) 40caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.debugdir = os.path.join(self.outputdir, 'debug') 414f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta os.mkdir(self.debugdir) 42b739f22a99c96d5a295f083125505b5b5ec2f8b6Achin Gupta if getpass.getuser() == 'root': 43b739f22a99c96d5a295f083125505b5b5ec2f8b6Achin Gupta self.configure_crash_handler() 444f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.bindir = bindir 454f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.srcdir = os.path.join(self.bindir, 'src') 464f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.tmpdir = tempfile.mkdtemp("_" + self.tagged_testname, 474f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta dir=job.tmpdir) 484f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self._keyvals = [] 494f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self._new_keyval = False 504f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.failed_constraints = [] 51caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.iteration = 0 52caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.before_iteration_hooks = [] 53caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.after_iteration_hooks = [] 54caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 55caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 56caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def configure_crash_handler(self): 57a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan pass 584f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 594f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 60caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def crash_handler_report(self): 61caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 62caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 63caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 64caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def assert_(self, expr, msg='Assertion failed.'): 654f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta if not expr: 66caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan raise error.TestError(msg) 67caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 68a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan 694f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def write_test_keyval(self, attr_dict): 704f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta utils.write_keyval(self.outputdir, attr_dict) 714f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 72caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 73caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @staticmethod 74a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan def _append_type_to_keys(dictionary, typename): 754f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta new_dict = {} 764f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta for key, value in dictionary.iteritems(): 774f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta new_key = "%s{%s}" % (key, typename) 78caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan new_dict[new_key] = value 79caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan return new_dict 80a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan 814f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 824f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def write_perf_keyval(self, perf_dict): 834f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.write_iteration_keyval({}, perf_dict) 844f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 854f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 864f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def write_attr_keyval(self, attr_dict): 874f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.write_iteration_keyval(attr_dict, {}) 88caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 89caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 90caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def write_iteration_keyval(self, attr_dict, perf_dict): 91caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # append the dictionaries before they have the {perf} and {attr} added 92caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self._keyvals.append({'attr':attr_dict, 'perf':perf_dict}) 93caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self._new_keyval = True 94caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 95caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if attr_dict: 96caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan attr_dict = self._append_type_to_keys(attr_dict, "attr") 97caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan utils.write_keyval(self.resultsdir, attr_dict, type_tag="attr") 98caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 99caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if perf_dict: 100caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan perf_dict = self._append_type_to_keys(perf_dict, "perf") 101a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan utils.write_keyval(self.resultsdir, perf_dict, type_tag="perf") 1024f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 103caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan keyval_path = os.path.join(self.resultsdir, "keyval") 104caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan print >> open(keyval_path, "a"), "" 105caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 106caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 107caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def analyze_perf_constraints(self, constraints): 108caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if not self._new_keyval: 1094f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta return 1104f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 111caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # create a dict from the keyvals suitable as an environment for eval 112a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan keyval_env = self._keyvals[-1]['perf'].copy() 113a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan keyval_env['__builtins__'] = None 1144f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self._new_keyval = False 1154f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta failures = [] 116caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 117a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan # evaluate each constraint using the current keyvals 118a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan for constraint in constraints: 1194f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta logging.info('___________________ constraint = %s', constraint) 1204f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta logging.info('___________________ keyvals = %s', keyval_env) 121caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 122a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan try: 1234f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta if not eval(constraint, keyval_env): 1244f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta failures.append('%s: constraint was not met' % constraint) 1254f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta except: 1264f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta failures.append('could not evaluate constraint: %s' 1274f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta % constraint) 1284f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 1294f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta # keep track of the errors for each iteration 130caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.failed_constraints.append(failures) 131caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 132caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 133caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def process_failed_constraints(self): 134caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan msg = '' 135caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan for i, failures in enumerate(self.failed_constraints): 136caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if failures: 137caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan msg += 'iteration %d:%s ' % (i, ','.join(failures)) 138caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 139a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan if msg: 1404f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta raise error.TestFail(msg) 1414f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 142caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 143caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def register_before_iteration_hook(self, iteration_hook): 144caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 145caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan This is how we expect test writers to register a before_iteration_hook. 146caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan This adds the method to the list of hooks which are executed 1474f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta before each iteration. 148caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 149caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param iteration_hook: Method to run before each iteration. A valid 150a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan hook accepts a single argument which is the 1514f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta test object. 1524f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta """ 1534f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.before_iteration_hooks.append(iteration_hook) 154caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 155caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 156a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan def register_after_iteration_hook(self, iteration_hook): 1574f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta """ 1584f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta This is how we expect test writers to register an after_iteration_hook. 1594f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta This adds the method to the list of hooks which are executed 160caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan after each iteration. 161caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 162a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan @param iteration_hook: Method to run after each iteration. A valid 1634f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta hook accepts a single argument which is the 1644f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta test object. 1654f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta """ 1664f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.after_iteration_hooks.append(iteration_hook) 1674f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 1684f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 1694f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def initialize(self): 170caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 171caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 172caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 173caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def setup(self): 174caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 175caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 176caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 177caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def warmup(self, *args, **dargs): 178caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 179a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan 1804f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 1814f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def drop_caches_between_iterations(self): 182caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if self.job.drop_caches_between_iterations: 183caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan print "Dropping caches between iterations" 184caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan utils.drop_caches() 185caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 186caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 1874f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta def _call_run_once(self, constraints, profile_only, 188caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan postprocess_profiled_run, args, dargs): 189caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.drop_caches_between_iterations() 190a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan 1914f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta # execute iteration hooks 1924f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta for hook in self.before_iteration_hooks: 1934f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta hook(self) 194caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 195caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if profile_only: 196a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan if not self.job.profilers.present(): 1974f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta self.job.record('WARN', None, None, 'No profilers have been ' 1984f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 'added but profile_only is set - nothing ' 1994f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta 'will be run') 200caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.run_once_profiling(postprocess_profiled_run, *args, **dargs) 201caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan else: 202a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan self.before_run_once() 203a7934d69508872919787742c7680b68807c24361Jeenu Viswambharan self.run_once(*args, **dargs) 204caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.after_run_once() 205caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 206caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan for hook in self.after_iteration_hooks: 207caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan hook(self) 208caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 209caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.postprocess_iteration() 210caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.analyze_perf_constraints(constraints) 211caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 212caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 213caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def execute(self, iterations=None, test_length=None, profile_only=None, 214caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _get_time=time.time, postprocess_profiled_run=None, 215caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan constraints=(), *args, **dargs): 216caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 217caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan This is the basic execute method for the tests inherited from base_test. 218caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan If you want to implement a benchmark test, it's better to implement 2190a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke the run_once function, to cope with the profiling infrastructure. For 220caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan other tests, you can just override the default implementation. 221caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 222caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param test_length: The minimum test length in seconds. We'll run the 223caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan run_once function for a number of times large enough to cover the 224caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan minimum test length. 225caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 226caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param iterations: A number of iterations that we'll run the run_once 227caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan function. This parameter is incompatible with test_length and will 228caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan be silently ignored if you specify both. 229caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 230caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param profile_only: If true run X iterations with profilers enabled. 231caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan If false run X iterations and one with profiling if profiles are 232caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan enabled. If None, default to the value of job.default_profile_only. 233caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 234caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param _get_time: [time.time] Used for unit test time injection. 235caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 236caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan @param postprocess_profiled_run: Run the postprocessing for the 237caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profiled run. 238caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 239caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 240caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # For our special class of tests, the benchmarks, we don't want 241caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # profilers to run during the test iterations. Let's reserve only 242caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # the last iteration for profiling, if needed. So let's stop 243caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # all profilers if they are present and active. 244caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers = self.job.profilers 245caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if profilers.active(): 246caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers.stop(self) 247caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if profile_only is None: 248caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profile_only = self.job.default_profile_only 249caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # If the user called this test in an odd way (specified both iterations 250caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # and test_length), let's warn them. 251caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if iterations and test_length: 252caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Iterations parameter ignored (timed execution).') 253caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if test_length: 254caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan test_start = _get_time() 255caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan time_elapsed = 0 256caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan timed_counter = 0 257caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Test started. Minimum test length: %d s', 258caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan test_length) 259caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan while time_elapsed < test_length: 260caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan timed_counter = timed_counter + 1 261caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if time_elapsed == 0: 262caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Executing iteration %d', timed_counter) 263caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan elif time_elapsed > 0: 264caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info( 265caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 'Executing iteration %d, time_elapsed %d s', 266caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan timed_counter, time_elapsed) 267caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self._call_run_once(constraints, profile_only, 268caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan postprocess_profiled_run, args, dargs) 269caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan test_iteration_finish = _get_time() 270caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan time_elapsed = test_iteration_finish - test_start 271caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Test finished after %d iterations', 272caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan timed_counter) 273caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Time elapsed: %d s', time_elapsed) 274caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan else: 275caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if iterations is None: 276caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan iterations = 1 277caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Test started. Number of iterations: %d', iterations) 278caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan for self.iteration in xrange(1, iterations+1): 279caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Executing iteration %d of %d', self.iteration, 280caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan iterations) 281caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self._call_run_once(constraints, profile_only, 282caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan postprocess_profiled_run, args, dargs) 283caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.info('Test finished after %d iterations.', iterations) 284caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 285caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if not profile_only: 286caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.iteration += 1 287caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.run_once_profiling(postprocess_profiled_run, *args, **dargs) 288caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 289caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Do any postprocessing, normally extracting performance keyvals, etc 290caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.postprocess() 291caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.process_failed_constraints() 292caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 293caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 294caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def run_once_profiling(self, postprocess_profiled_run, *args, **dargs): 295caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers = self.job.profilers 296caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Do a profiling run if necessary 297caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if profilers.present(): 298caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.drop_caches_between_iterations() 299caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers.before_start(self) 300caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 301caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.before_run_once() 302caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers.start(self) 303caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan print 'Profilers present. Profiling run started' 304caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 305caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 306caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.run_once(*args, **dargs) 307caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 308caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Priority to the run_once() argument over the attribute. 309caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan postprocess_attribute = getattr(self, 310caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 'postprocess_profiled_run', 311caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan False) 312caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 313caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if (postprocess_profiled_run or 314caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan (postprocess_profiled_run is None and 315caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan postprocess_attribute)): 316caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.postprocess_iteration() 317caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 318caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan finally: 319caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers.stop(self) 320caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan profilers.report(self) 321caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 322caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.after_run_once() 323caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 324caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 325caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def postprocess(self): 326caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 327caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 328caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 329caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def postprocess_iteration(self): 330caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 331caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 332caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 3330a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke def cleanup(self): 3340a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke pass 3350a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke 336caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 337caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def before_run_once(self): 338caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 339caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan Override in tests that need it, will be called before any run_once() 340caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan call including the profiling run (when it's called before starting 341caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan the profilers). 342caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 343caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 344caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 345caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 346caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def after_run_once(self): 347caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 348caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan Called after every run_once (including from a profiled run when it's 349caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan called after stopping the profilers). 350caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan """ 351caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass 352caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 353caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 354caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan def _exec(self, args, dargs): 355caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.job.logging.tee_redirect_debug_dir(self.debugdir, 356caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan log_name=self.tagged_testname) 357caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 358caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if self.network_destabilizing: 359caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.job.disable_warnings("NETWORK") 360caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 361caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # write out the test attributes into a keyval 3624f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta dargs = dargs.copy() 3634f6ad66ae9fcc8bcb3b0fcee10b7ab1ffcaf1a5Achin Gupta run_cleanup = dargs.pop('run_cleanup', self.job.run_test_cleanup) 364caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan keyvals = dargs.pop('test_attributes', {}).copy() 365caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan keyvals['version'] = self.version 366caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan for i, arg in enumerate(args): 367caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan keyvals['param-%d' % i] = repr(arg) 368caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan for name, arg in dargs.iteritems(): 369caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan keyvals['param-%s' % name] = repr(arg) 370caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.write_test_keyval(keyvals) 371caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 372caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _validate_args(args, dargs, self.initialize, self.setup, 373caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.execute, self.cleanup) 374caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 375caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 376caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Initialize: 377caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _cherry_pick_call(self.initialize, *args, **dargs) 378caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 379caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan lockfile = open(os.path.join(self.job.tmpdir, '.testlock'), 'w') 380caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 381caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan fcntl.flock(lockfile, fcntl.LOCK_EX) 382caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Setup: (compile and install the test, if needed) 383caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan p_args, p_dargs = _cherry_pick_args(self.setup,args,dargs) 384caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan utils.update_version(self.srcdir, self.preserve_srcdir, 385caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.version, self.setup, 386caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan *p_args, **p_dargs) 387caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan finally: 388caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan fcntl.flock(lockfile, fcntl.LOCK_UN) 3890a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke lockfile.close() 390caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 391caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Execute: 392caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan os.chdir(self.outputdir) 393caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 394caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # call self.warmup cherry picking the arguments it accepts and 395caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # translate exceptions if needed 396caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _call_test_function(_cherry_pick_call, self.warmup, 397caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan *args, **dargs) 398caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 399caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if hasattr(self, 'run_once'): 400caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan p_args, p_dargs = _cherry_pick_args(self.run_once, 401caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan args, dargs) 402caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # pull in any non-* and non-** args from self.execute 4030a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke for param in _get_nonstar_args(self.execute): 404caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if param in dargs: 405caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan p_dargs[param] = dargs[param] 406caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan else: 407caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan p_args, p_dargs = _cherry_pick_args(self.execute, 408caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan args, dargs) 409caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 410caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _call_test_function(self.execute, *p_args, **p_dargs) 411caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan except Exception: 412caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 413caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan logging.exception('Exception escaping from test:') 414caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan except: 415caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan pass # don't let logging exceptions here interfere 416caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan 417caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Save the exception while we run our cleanup() before 418caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # reraising it. 419caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan exc_info = sys.exc_info() 420caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 421caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 422caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if run_cleanup: 423caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _cherry_pick_call(self.cleanup, *args, **dargs) 424caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan except Exception: 425caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan print 'Ignoring exception during cleanup() phase:' 426caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan traceback.print_exc() 427caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan print 'Now raising the earlier %s error' % exc_info[0] 428caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.crash_handler_report() 429caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan finally: 430caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.job.logging.restore() 431caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan try: 432caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan raise exc_info[0], exc_info[1], exc_info[2] 433caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan finally: 434caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # http://docs.python.org/library/sys.html#sys.exc_info 435caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan # Be nice and prevent a circular reference. 4360a30cf54af7bb1f77b405062b1d5b44e809d0290Andrew Thoelke del exc_info 437caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan else: 4382bf28e620a6f05700753a2b45a888c6623e20723Andrew Thoelke try: 439caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan if run_cleanup: 440caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan _cherry_pick_call(self.cleanup, *args, **dargs) 441caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.crash_handler_report() 442caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan finally: 443caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan self.job.logging.restore() 444caa84939a4d8b1189dea8619ccc57bdb3026b125Jeenu Viswambharan except error.AutotestError: 4452bf28e620a6f05700753a2b45a888c6623e20723Andrew Thoelke if self.network_destabilizing: 4462bf28e620a6f05700753a2b45a888c6623e20723Andrew Thoelke self.job.enable_warnings("NETWORK") 447 # Pass already-categorized errors on up. 448 raise 449 except Exception, e: 450 if self.network_destabilizing: 451 self.job.enable_warnings("NETWORK") 452 # Anything else is an ERROR in our own code, not execute(). 453 raise error.UnhandledTestError(e) 454 else: 455 if self.network_destabilizing: 456 self.job.enable_warnings("NETWORK") 457 458 459def _get_nonstar_args(func): 460 """Extract all the (normal) function parameter names. 461 462 Given a function, returns a tuple of parameter names, specifically 463 excluding the * and ** parameters, if the function accepts them. 464 465 @param func: A callable that we want to chose arguments for. 466 467 @return: A tuple of parameters accepted by the function. 468 """ 469 return func.func_code.co_varnames[:func.func_code.co_argcount] 470 471 472def _cherry_pick_args(func, args, dargs): 473 """Sanitize positional and keyword arguments before calling a function. 474 475 Given a callable (func), an argument tuple and a dictionary of keyword 476 arguments, pick only those arguments which the function is prepared to 477 accept and return a new argument tuple and keyword argument dictionary. 478 479 Args: 480 func: A callable that we want to choose arguments for. 481 args: A tuple of positional arguments to consider passing to func. 482 dargs: A dictionary of keyword arguments to consider passing to func. 483 Returns: 484 A tuple of: (args tuple, keyword arguments dictionary) 485 """ 486 # Cherry pick args: 487 if func.func_code.co_flags & 0x04: 488 # func accepts *args, so return the entire args. 489 p_args = args 490 else: 491 p_args = () 492 493 # Cherry pick dargs: 494 if func.func_code.co_flags & 0x08: 495 # func accepts **dargs, so return the entire dargs. 496 p_dargs = dargs 497 else: 498 # Only return the keyword arguments that func accepts. 499 p_dargs = {} 500 for param in _get_nonstar_args(func): 501 if param in dargs: 502 p_dargs[param] = dargs[param] 503 504 return p_args, p_dargs 505 506 507def _cherry_pick_call(func, *args, **dargs): 508 """Cherry picks arguments from args/dargs based on what "func" accepts 509 and calls the function with the picked arguments.""" 510 p_args, p_dargs = _cherry_pick_args(func, args, dargs) 511 return func(*p_args, **p_dargs) 512 513 514def _validate_args(args, dargs, *funcs): 515 """Verify that arguments are appropriate for at least one callable. 516 517 Given a list of callables as additional parameters, verify that 518 the proposed keyword arguments in dargs will each be accepted by at least 519 one of the callables. 520 521 NOTE: args is currently not supported and must be empty. 522 523 Args: 524 args: A tuple of proposed positional arguments. 525 dargs: A dictionary of proposed keyword arguments. 526 *funcs: Callables to be searched for acceptance of args and dargs. 527 Raises: 528 error.AutotestError: if an arg won't be accepted by any of *funcs. 529 """ 530 all_co_flags = 0 531 all_varnames = () 532 for func in funcs: 533 all_co_flags |= func.func_code.co_flags 534 all_varnames += func.func_code.co_varnames[:func.func_code.co_argcount] 535 536 # Check if given args belongs to at least one of the methods below. 537 if len(args) > 0: 538 # Current implementation doesn't allow the use of args. 539 raise error.TestError('Unnamed arguments not accepted. Please ' 540 'call job.run_test with named args only') 541 542 # Check if given dargs belongs to at least one of the methods below. 543 if len(dargs) > 0: 544 if not all_co_flags & 0x08: 545 # no func accepts *dargs, so: 546 for param in dargs: 547 if not param in all_varnames: 548 raise error.AutotestError('Unknown parameter: %s' % param) 549 550 551def _installtest(job, url): 552 (group, name) = job.pkgmgr.get_package_name(url, 'test') 553 554 # Bail if the test is already installed 555 group_dir = os.path.join(job.testdir, "download", group) 556 if os.path.exists(os.path.join(group_dir, name)): 557 return (group, name) 558 559 # If the group directory is missing create it and add 560 # an empty __init__.py so that sub-directories are 561 # considered for import. 562 if not os.path.exists(group_dir): 563 os.mkdir(group_dir) 564 f = file(os.path.join(group_dir, '__init__.py'), 'w+') 565 f.close() 566 567 print name + ": installing test url=" + url 568 tarball = os.path.basename(url) 569 tarball_path = os.path.join(group_dir, tarball) 570 test_dir = os.path.join(group_dir, name) 571 job.pkgmgr.fetch_pkg(tarball, tarball_path, 572 repo_url = os.path.dirname(url)) 573 574 # Create the directory for the test 575 if not os.path.exists(test_dir): 576 os.mkdir(os.path.join(group_dir, name)) 577 578 job.pkgmgr.untar_pkg(tarball_path, test_dir) 579 580 os.remove(tarball_path) 581 582 # For this 'sub-object' to be importable via the name 583 # 'group.name' we need to provide an __init__.py, 584 # so link the main entry point to this. 585 os.symlink(name + '.py', os.path.join(group_dir, name, 586 '__init__.py')) 587 588 # The test is now installed. 589 return (group, name) 590 591 592def _call_test_function(func, *args, **dargs): 593 """Calls a test function and translates exceptions so that errors 594 inside test code are considered test failures.""" 595 try: 596 return func(*args, **dargs) 597 except error.AutotestError: 598 # Pass already-categorized errors on up as is. 599 raise 600 except Exception, e: 601 # Other exceptions must be treated as a FAIL when 602 # raised during the test functions 603 raise error.UnhandledTestFail(e) 604 605 606def runtest(job, url, tag, args, dargs, 607 local_namespace={}, global_namespace={}, 608 before_test_hook=None, after_test_hook=None, 609 before_iteration_hook=None, after_iteration_hook=None): 610 local_namespace = local_namespace.copy() 611 global_namespace = global_namespace.copy() 612 613 # if this is not a plain test name then download and install the 614 # specified test 615 if url.endswith('.tar.bz2'): 616 (group, testname) = _installtest(job, url) 617 bindir = os.path.join(job.testdir, 'download', group, testname) 618 site_bindir = None 619 else: 620 # if the test is local, it can be found in either testdir 621 # or site_testdir. tests in site_testdir override tests 622 # defined in testdir 623 (group, testname) = ('', url) 624 bindir = os.path.join(job.testdir, group, testname) 625 if hasattr(job, 'site_testdir'): 626 site_bindir = os.path.join(job.site_testdir, 627 group, testname) 628 else: 629 site_bindir = None 630 631 # The job object here can be that of a server side job or a client 632 # side job. 'install_pkg' method won't be present for server side 633 # jobs, so do the fetch only if that method is present in the job 634 # obj. 635 if hasattr(job, 'install_pkg'): 636 try: 637 job.install_pkg(testname, 'test', bindir) 638 except error.PackageInstallError, e: 639 # continue as a fall back mechanism and see if the test code 640 # already exists on the machine 641 pass 642 643 outputdir = os.path.join(job.resultdir, testname) 644 if tag: 645 outputdir += '.' + tag 646 647 # if we can find the test in site_bindir, use this version 648 if site_bindir and os.path.exists(site_bindir): 649 bindir = site_bindir 650 testdir = job.site_testdir 651 elif os.path.exists(bindir): 652 testdir = job.testdir 653 else: 654 raise error.TestError(testname + ': test does not exist') 655 656 local_namespace['job'] = job 657 local_namespace['bindir'] = bindir 658 local_namespace['outputdir'] = outputdir 659 660 if group: 661 sys.path.insert(0, os.path.join(testdir, 'download')) 662 group += '.' 663 else: 664 sys.path.insert(0, os.path.join(testdir, testname)) 665 666 try: 667 exec ("import %s%s" % (group, testname), 668 local_namespace, global_namespace) 669 exec ("mytest = %s%s.%s(job, bindir, outputdir)" % 670 (group, testname, testname), 671 local_namespace, global_namespace) 672 finally: 673 sys.path.pop(0) 674 675 pwd = os.getcwd() 676 os.chdir(outputdir) 677 678 try: 679 mytest = global_namespace['mytest'] 680 if before_test_hook: 681 before_test_hook(mytest) 682 683 # we use the register iteration hooks methods to register the passed 684 # in hooks 685 if before_iteration_hook: 686 mytest.register_before_iteration_hook(before_iteration_hook) 687 if after_iteration_hook: 688 mytest.register_after_iteration_hook(after_iteration_hook) 689 mytest._exec(args, dargs) 690 finally: 691 os.chdir(pwd) 692 if after_test_hook: 693 after_test_hook(mytest) 694 shutil.rmtree(mytest.tmpdir, ignore_errors=True) 695