19aec6930955ea7d3d345cf2d214f9e6982f84cb8Richard Barnette# pylint: disable=missing-docstring 29aec6930955ea7d3d345cf2d214f9e6982f84cb8Richard Barnette 34afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanskiimport cPickle as pickle 4f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport copy 5f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport errno 6f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport fcntl 7f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport logging 8f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport os 9f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport re 10d9a056fe72b1a6aa2ccf9acbc3a1918dbbe07028Fang Dengimport tempfile 11f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport time 12f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport traceback 13f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshetimport weakref 144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanskifrom autotest_lib.client.common_lib import autotemp, error, log 15da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 16da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 17da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanskiclass job_directory(object): 18da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Represents a job.*dir directory.""" 19da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 20da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 21fc3da5bbdcd63840cd224282812c050c5520db11mbligh class JobDirectoryException(error.AutotestError): 22fc3da5bbdcd63840cd224282812c050c5520db11mbligh """Generic job_directory exception superclass.""" 23fc3da5bbdcd63840cd224282812c050c5520db11mbligh 24fc3da5bbdcd63840cd224282812c050c5520db11mbligh 25fc3da5bbdcd63840cd224282812c050c5520db11mbligh class MissingDirectoryException(JobDirectoryException): 26da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a directory required by the job does not exist.""" 27da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path): 28da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, 'Directory %s does not exist' % path) 29da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 30da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 31fc3da5bbdcd63840cd224282812c050c5520db11mbligh class UncreatableDirectoryException(JobDirectoryException): 32da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a directory required by the job is missing and cannot 33da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski be created.""" 34da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path, error): 35da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg = 'Creation of directory %s failed with exception %s' 36da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg %= (path, error) 37da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, msg) 38da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 39da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 40fc3da5bbdcd63840cd224282812c050c5520db11mbligh class UnwritableDirectoryException(JobDirectoryException): 41da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a writable directory required by the job exists 42da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski but is not writable.""" 43da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path): 44da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg = 'Directory %s exists but is not writable' % path 45da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, msg) 46da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 47da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 48da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path, is_writable=False): 49da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 50da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Instantiate a job directory. 51da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 524afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param path: The path of the directory. If None a temporary directory 53da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski will be created instead. 544afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param is_writable: If True, expect the directory to be writable. 55da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 564afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise MissingDirectoryException: raised if is_writable=False and the 57da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist. 584afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise UnwritableDirectoryException: raised if is_writable=True and 59da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the directory exists but is not writable. 604afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise UncreatableDirectoryException: raised if is_writable=True, the 61da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist and it cannot be created. 62da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 63da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if path is None: 64da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if is_writable: 65da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._tempdir = autotemp.tempdir(unique_id='autotest') 66da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self.path = self._tempdir.name 67da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 68da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.MissingDirectoryException(path) 69da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 70da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._tempdir = None 71da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self.path = path 72da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._ensure_valid(is_writable) 73da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 74da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 75da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _ensure_valid(self, is_writable): 76da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 77da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Ensure that this is a valid directory. 78da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 79da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Will check if a directory exists, can optionally also enforce that 80da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski it be writable. It can optionally create it if necessary. Creation 81da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski will still fail if the path is rooted in a non-writable directory, or 82da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if a file already exists at the given location. 83da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 84da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param dir_path A path where a directory should be located 85da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param is_writable A boolean indicating that the directory should 86da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski not only exist, but also be writable. 87da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 88da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises MissingDirectoryException raised if is_writable=False and the 89da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist. 90da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UnwritableDirectoryException raised if is_writable=True and 91da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the directory is not wrtiable. 92da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UncreatableDirectoryException raised if is_writable=True, the 93da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist and it cannot be created 94da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 958054b0da315c94d249aaf80512bc133d4cb55b92mbligh # ensure the directory exists 968054b0da315c94d249aaf80512bc133d4cb55b92mbligh if is_writable: 978054b0da315c94d249aaf80512bc133d4cb55b92mbligh try: 988054b0da315c94d249aaf80512bc133d4cb55b92mbligh os.makedirs(self.path) 998054b0da315c94d249aaf80512bc133d4cb55b92mbligh except OSError, e: 100fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if e.errno != errno.EEXIST or not os.path.isdir(self.path): 101da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.UncreatableDirectoryException(self.path, e) 1028054b0da315c94d249aaf80512bc133d4cb55b92mbligh elif not os.path.isdir(self.path): 1038054b0da315c94d249aaf80512bc133d4cb55b92mbligh raise self.MissingDirectoryException(self.path) 104da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 105da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # if is_writable=True, also check that the directory is writable 106da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if is_writable and not os.access(self.path, os.W_OK): 107da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.UnwritableDirectoryException(self.path) 108da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 109da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 110da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @staticmethod 111da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def property_factory(attribute): 112da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 113da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Create a job.*dir -> job._*dir.path property accessor. 114da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 115da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param attribute A string with the name of the attribute this is 116da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski exposed as. '_'+attribute must then be attribute that holds 117da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski either None or a job_directory-like object. 118da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 119da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @returns A read-only property object that exposes a job_directory path 120da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 121da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @property 122da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def dir_property(self): 123da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski underlying_attribute = getattr(self, '_' + attribute) 124da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if underlying_attribute is None: 125da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return None 126da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 127da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return underlying_attribute.path 128da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return dir_property 129da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 130da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 131a2c994965280e84061c5e0502b711947f780bb02mbligh# decorator for use with job_state methods 132a2c994965280e84061c5e0502b711947f780bb02mblighdef with_backing_lock(method): 133a2c994965280e84061c5e0502b711947f780bb02mbligh """A decorator to perform a lock-*-unlock cycle. 134a2c994965280e84061c5e0502b711947f780bb02mbligh 135a2c994965280e84061c5e0502b711947f780bb02mbligh When applied to a method, this decorator will automatically wrap 136a2c994965280e84061c5e0502b711947f780bb02mbligh calls to the method in a backing file lock and before the call 137a2c994965280e84061c5e0502b711947f780bb02mbligh followed by a backing file unlock. 138a2c994965280e84061c5e0502b711947f780bb02mbligh """ 139a2c994965280e84061c5e0502b711947f780bb02mbligh def wrapped_method(self, *args, **dargs): 140a2c994965280e84061c5e0502b711947f780bb02mbligh already_have_lock = self._backing_file_lock is not None 141a2c994965280e84061c5e0502b711947f780bb02mbligh if not already_have_lock: 142a2c994965280e84061c5e0502b711947f780bb02mbligh self._lock_backing_file() 143a2c994965280e84061c5e0502b711947f780bb02mbligh try: 144a2c994965280e84061c5e0502b711947f780bb02mbligh return method(self, *args, **dargs) 145a2c994965280e84061c5e0502b711947f780bb02mbligh finally: 146a2c994965280e84061c5e0502b711947f780bb02mbligh if not already_have_lock: 147a2c994965280e84061c5e0502b711947f780bb02mbligh self._unlock_backing_file() 148a2c994965280e84061c5e0502b711947f780bb02mbligh wrapped_method.__name__ = method.__name__ 149a2c994965280e84061c5e0502b711947f780bb02mbligh wrapped_method.__doc__ = method.__doc__ 150a2c994965280e84061c5e0502b711947f780bb02mbligh return wrapped_method 151a2c994965280e84061c5e0502b711947f780bb02mbligh 152a2c994965280e84061c5e0502b711947f780bb02mbligh 153a2c994965280e84061c5e0502b711947f780bb02mbligh# decorator for use with job_state methods 154a2c994965280e84061c5e0502b711947f780bb02mblighdef with_backing_file(method): 155a2c994965280e84061c5e0502b711947f780bb02mbligh """A decorator to perform a lock-read-*-write-unlock cycle. 156a2c994965280e84061c5e0502b711947f780bb02mbligh 157a2c994965280e84061c5e0502b711947f780bb02mbligh When applied to a method, this decorator will automatically wrap 158a2c994965280e84061c5e0502b711947f780bb02mbligh calls to the method in a lock-and-read before the call followed by a 159a2c994965280e84061c5e0502b711947f780bb02mbligh write-and-unlock. Any operation that is reading or writing state 160a2c994965280e84061c5e0502b711947f780bb02mbligh should be decorated with this method to ensure that backing file 161a2c994965280e84061c5e0502b711947f780bb02mbligh state is consistently maintained. 162a2c994965280e84061c5e0502b711947f780bb02mbligh """ 163a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_lock 164a2c994965280e84061c5e0502b711947f780bb02mbligh def wrapped_method(self, *args, **dargs): 165a2c994965280e84061c5e0502b711947f780bb02mbligh self._read_from_backing_file() 166a2c994965280e84061c5e0502b711947f780bb02mbligh try: 167a2c994965280e84061c5e0502b711947f780bb02mbligh return method(self, *args, **dargs) 168a2c994965280e84061c5e0502b711947f780bb02mbligh finally: 169a2c994965280e84061c5e0502b711947f780bb02mbligh self._write_to_backing_file() 170a2c994965280e84061c5e0502b711947f780bb02mbligh wrapped_method.__name__ = method.__name__ 171a2c994965280e84061c5e0502b711947f780bb02mbligh wrapped_method.__doc__ = method.__doc__ 172a2c994965280e84061c5e0502b711947f780bb02mbligh return wrapped_method 173a2c994965280e84061c5e0502b711947f780bb02mbligh 174a2c994965280e84061c5e0502b711947f780bb02mbligh 175a2c994965280e84061c5e0502b711947f780bb02mbligh 176fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mblighclass job_state(object): 177fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """A class for managing explicit job and user state, optionally persistent. 178fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 179fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh The class allows you to save state by name (like a dictionary). Any state 180fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh stored in this class should be picklable and deep copyable. While this is 181fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh not enforced it is recommended that only valid python identifiers be used 182fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh as names. Additionally, the namespace 'stateful_property' is used for 183fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh storing the valued associated with properties constructed using the 184fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh property_factory method. 185fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 186fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 187fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh NO_DEFAULT = object() 188fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh PICKLE_PROTOCOL = 2 # highest protocol available in python 2.4 189fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 190fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 191fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def __init__(self): 192fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Initialize the job state.""" 193fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state = {} 194fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._backing_file = None 195fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_initialized = False 196fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_lock = None 197fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 198fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 199fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def _lock_backing_file(self): 200fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """Acquire a lock on the backing file.""" 201fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if self._backing_file: 202fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_lock = open(self._backing_file, 'a') 203a087eaedccf3bf2a89808f22254d282bd7147f8cjadmanski fcntl.flock(self._backing_file_lock, fcntl.LOCK_EX) 204fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 205fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 206fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def _unlock_backing_file(self): 207fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """Release a lock on the backing file.""" 208fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if self._backing_file_lock: 209a087eaedccf3bf2a89808f22254d282bd7147f8cjadmanski fcntl.flock(self._backing_file_lock, fcntl.LOCK_UN) 210fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_lock.close() 211fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_lock = None 212fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 213fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 214fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def read_from_file(self, file_path, merge=True): 215fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Read in any state from the file at file_path. 216fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 217fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski When merge=True, any state specified only in-memory will be preserved. 218fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski Any state specified on-disk will be set in-memory, even if an in-memory 219fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski setting already exists. 220fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 2214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param file_path: The path where the state should be read from. It must 222fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski exist but it can be empty. 2234afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param merge: If true, merge the on-disk state with the in-memory 224fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski state. If false, replace the in-memory state with the on-disk 225fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski state. 226fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 2274afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @warning: This method is intentionally concurrency-unsafe. It makes no 228fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski attempt to control concurrent access to the file at file_path. 229fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 230fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 231fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski # we can assume that the file exists 232fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if os.path.getsize(file_path) == 0: 233fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski on_disk_state = {} 234fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 235fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski on_disk_state = pickle.load(open(file_path)) 236fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 237fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if merge: 238fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski # merge the on-disk state with the in-memory state 239fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski for namespace, namespace_dict in on_disk_state.iteritems(): 240fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski in_memory_namespace = self._state.setdefault(namespace, {}) 241fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski for name, value in namespace_dict.iteritems(): 242fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if name in in_memory_namespace: 243fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if in_memory_namespace[name] != value: 244fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski logging.info('Persistent value of %s.%s from %s ' 245fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 'overridding existing in-memory ' 246fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 'value', namespace, name, file_path) 247fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski in_memory_namespace[name] = value 248fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski else: 249fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski logging.debug('Value of %s.%s is unchanged, ' 250fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 'skipping import', namespace, name) 251fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 252fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski logging.debug('Importing %s.%s from state file %s', 253fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski namespace, name, file_path) 254fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski in_memory_namespace[name] = value 255fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski else: 256fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski # just replace the in-memory state with the on-disk state 257fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._state = on_disk_state 258fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 259a2c994965280e84061c5e0502b711947f780bb02mbligh # lock the backing file before we refresh it 260a2c994965280e84061c5e0502b711947f780bb02mbligh with_backing_lock(self.__class__._write_to_backing_file)(self) 261fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 262fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 263fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def write_to_file(self, file_path): 264fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Write out the current state to the given path. 265fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 2664afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param file_path: The path where the state should be written out to. 267fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh Must be writable. 268fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 2694afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @warning: This method is intentionally concurrency-unsafe. It makes no 270a2c994965280e84061c5e0502b711947f780bb02mbligh attempt to control concurrent access to the file at file_path. 271fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 272fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh outfile = open(file_path, 'w') 273fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh try: 274fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh pickle.dump(self._state, outfile, self.PICKLE_PROTOCOL) 275fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh finally: 276fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh outfile.close() 277fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 278fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 279fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def _read_from_backing_file(self): 280fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """Refresh the current state from the backing file. 281fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 282fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski If the backing file has never been read before (indicated by checking 283fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_initialized) it will merge the file with the 284fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski in-memory state, rather than overwriting it. 285fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """ 286fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if self._backing_file: 287fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski merge_backing_file = not self._backing_file_initialized 288fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self.read_from_file(self._backing_file, merge=merge_backing_file) 289fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_initialized = True 290fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 291fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 292fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def _write_to_backing_file(self): 293fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Flush the current state to the backing file.""" 294fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if self._backing_file: 295fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self.write_to_file(self._backing_file) 296fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 297fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 298a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 299fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def _synchronize_backing_file(self): 300fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """Synchronizes the contents of the in-memory and on-disk state.""" 301fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski # state is implicitly synchronized in _with_backing_file methods 302fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski pass 303fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 304fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 305fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set_backing_file(self, file_path): 306fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Change the path used as the backing file for the persistent state. 307fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 308fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh When a new backing file is specified if a file already exists then 309fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh its contents will be added into the current state, with conflicts 310fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh between the file and memory being resolved in favor of the file 311fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh contents. The file will then be kept in sync with the (combined) 312fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh in-memory state. The syncing can be disabled by setting this to None. 313fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 3144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param file_path: A path on the filesystem that can be read from and 315fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh written to, or None to turn off the backing store. 316fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 317fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._synchronize_backing_file() 318fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._backing_file = file_path 319fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._backing_file_initialized = False 320fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski self._synchronize_backing_file() 321fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 322fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 323a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 324fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski def get(self, namespace, name, default=NO_DEFAULT): 325fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """Returns the value associated with a particular name. 326fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 3274afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace that the property should be stored in. 3284afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name the value was saved with. 3294afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param default: A default value to return if no state is currently 330fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski associated with var. 331fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski 3324afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A deep copy of the value associated with name. Note that this 333fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski explicitly returns a deep copy to avoid problems with mutable 334fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski values; mutations are not persisted or shared. 3354afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise KeyError: raised when no state is associated with var and a 336fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski default value is not provided. 337fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski """ 338fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski if self.has(namespace, name): 339fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski return copy.deepcopy(self._state[namespace][name]) 340fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski elif default is self.NO_DEFAULT: 341fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski raise KeyError('No key %s in namespace %s' % (name, namespace)) 342fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski else: 343fa2e889be2e704a8b3c723900f26cfe33a9cbb7djadmanski return default 344fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 345fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 346a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 347fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set(self, namespace, name, value): 348fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Saves the value given with the provided name. 349fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 3504afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace that the property should be stored in. 3514afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name the value should be saved with. 3524afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param value: The value to save. 353fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 354fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace_dict = self._state.setdefault(namespace, {}) 355fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace_dict[name] = copy.deepcopy(value) 356fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Persistent state %s.%s now set to %r', namespace, 357fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh name, value) 358fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 359fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 360a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 361fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def has(self, namespace, name): 362fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Return a boolean indicating if namespace.name is defined. 363fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 3644afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace to check for a definition. 3654afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name to check for a definition. 366fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 3674afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: True if the given name is defined in the given namespace and 368fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh False otherwise. 369fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 370fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return namespace in self._state and name in self._state[namespace] 371fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 372fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 373a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 374fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def discard(self, namespace, name): 375fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """If namespace.name is a defined value, deletes it. 376fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 3774afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace that the property is stored in. 3784afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name the value is saved with. 379fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 380fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if self.has(namespace, name): 381fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh del self._state[namespace][name] 382fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if len(self._state[namespace]) == 0: 383fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh del self._state[namespace] 384fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Persistent state %s.%s deleted', namespace, name) 385fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 386fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug( 387fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 'Persistent state %s.%s not defined so nothing is discarded', 388fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace, name) 389fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 390fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 391a2c994965280e84061c5e0502b711947f780bb02mbligh @with_backing_file 392fc3da5bbdcd63840cd224282812c050c5520db11mbligh def discard_namespace(self, namespace): 393fc3da5bbdcd63840cd224282812c050c5520db11mbligh """Delete all defined namespace.* names. 394fc3da5bbdcd63840cd224282812c050c5520db11mbligh 3954afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace to be cleared. 396fc3da5bbdcd63840cd224282812c050c5520db11mbligh """ 397fc3da5bbdcd63840cd224282812c050c5520db11mbligh if namespace in self._state: 398fc3da5bbdcd63840cd224282812c050c5520db11mbligh del self._state[namespace] 399fc3da5bbdcd63840cd224282812c050c5520db11mbligh logging.debug('Persistent state %s.* deleted', namespace) 400fc3da5bbdcd63840cd224282812c050c5520db11mbligh 401fc3da5bbdcd63840cd224282812c050c5520db11mbligh 402fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @staticmethod 403fc3da5bbdcd63840cd224282812c050c5520db11mbligh def property_factory(state_attribute, property_attribute, default, 404fc3da5bbdcd63840cd224282812c050c5520db11mbligh namespace='global_properties'): 405fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 406fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh Create a property object for an attribute using self.get and self.set. 407fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 4084afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param state_attribute: A string with the name of the attribute on 409fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh job that contains the job_state instance. 4104afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param property_attribute: A string with the name of the attribute 411fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh this property is exposed as. 4124afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param default: A default value that should be used for this property 413fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if it is not set. 4144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param namespace: The namespace to store the attribute value in. 415fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 4164afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A read-write property object that performs self.get calls 417fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh to read the value and self.set calls to set it. 418fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 419fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def getter(job): 420fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh state = getattr(job, state_attribute) 421fc3da5bbdcd63840cd224282812c050c5520db11mbligh return state.get(namespace, property_attribute, default) 422fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def setter(job, value): 423fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh state = getattr(job, state_attribute) 424fc3da5bbdcd63840cd224282812c050c5520db11mbligh state.set(namespace, property_attribute, value) 425fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return property(getter, setter) 426fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 427fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 4284afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanskiclass status_log_entry(object): 4294afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Represents a single status log entry.""" 4304afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 4312a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski RENDERED_NONE_VALUE = '----' 4322a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski TIMESTAMP_FIELD = 'timestamp' 4332a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski LOCALTIME_FIELD = 'localtime' 4342a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 435861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li # non-space whitespace is forbidden in any fields 436861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li BAD_CHAR_REGEX = re.compile(r'[\t\n\r\v\f]') 437861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li 4386ae02f0988f60b893b47a5059738060ca3773b02xixuan def _init_message(self, message): 4396ae02f0988f60b893b47a5059738060ca3773b02xixuan """Handle the message which describs event to be recorded. 4406ae02f0988f60b893b47a5059738060ca3773b02xixuan 4416ae02f0988f60b893b47a5059738060ca3773b02xixuan Break the message line into a single-line message that goes into the 4426ae02f0988f60b893b47a5059738060ca3773b02xixuan database, and a block of additional lines that goes into the status 4436ae02f0988f60b893b47a5059738060ca3773b02xixuan log but will never be parsed 4446ae02f0988f60b893b47a5059738060ca3773b02xixuan When detecting a bad char in message, replace it with space instead 4456ae02f0988f60b893b47a5059738060ca3773b02xixuan of raising an exception that cannot be parsed by tko parser. 4466ae02f0988f60b893b47a5059738060ca3773b02xixuan 4476ae02f0988f60b893b47a5059738060ca3773b02xixuan @param message: the input message. 4486ae02f0988f60b893b47a5059738060ca3773b02xixuan 4496ae02f0988f60b893b47a5059738060ca3773b02xixuan @return: filtered message without bad characters. 4506ae02f0988f60b893b47a5059738060ca3773b02xixuan """ 4516ae02f0988f60b893b47a5059738060ca3773b02xixuan message_lines = message.splitlines() 4526ae02f0988f60b893b47a5059738060ca3773b02xixuan if message_lines: 4536ae02f0988f60b893b47a5059738060ca3773b02xixuan self.message = message_lines[0] 4546ae02f0988f60b893b47a5059738060ca3773b02xixuan self.extra_message_lines = message_lines[1:] 4556ae02f0988f60b893b47a5059738060ca3773b02xixuan else: 4566ae02f0988f60b893b47a5059738060ca3773b02xixuan self.message = '' 4576ae02f0988f60b893b47a5059738060ca3773b02xixuan self.extra_message_lines = [] 4586ae02f0988f60b893b47a5059738060ca3773b02xixuan 4596ae02f0988f60b893b47a5059738060ca3773b02xixuan self.message = self.message.replace('\t', ' ' * 8) 4606ae02f0988f60b893b47a5059738060ca3773b02xixuan self.message = self.BAD_CHAR_REGEX.sub(' ', self.message) 4616ae02f0988f60b893b47a5059738060ca3773b02xixuan 4626ae02f0988f60b893b47a5059738060ca3773b02xixuan 4634afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def __init__(self, status_code, subdir, operation, message, fields, 4644afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski timestamp=None): 4654afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Construct a status.log entry. 4664afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 4674afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param status_code: A message status code. Must match the codes 4684afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski accepted by autotest_lib.common_lib.log.is_valid_status. 4694afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param subdir: A valid job subdirectory, or None. 4704afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param operation: Description of the operation, or None. 4714afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param message: A printable string describing event to be recorded. 4724afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param fields: A dictionary of arbitrary alphanumeric key=value pairs 4734afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski to be included in the log, or None. 4744afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param timestamp: An optional integer timestamp, in the same format 4754afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski as a time.time() timestamp. If unspecified, the current time is 4764afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski used. 4774afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 4784afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise ValueError: if any of the parameters are invalid 4794afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 4804afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if not log.is_valid_status(status_code): 4814afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise ValueError('status code %r is not valid' % status_code) 4824afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.status_code = status_code 4834afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 484861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li if subdir and self.BAD_CHAR_REGEX.search(subdir): 4854afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise ValueError('Invalid character in subdir string') 4864afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.subdir = subdir 4874afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 488861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li if operation and self.BAD_CHAR_REGEX.search(operation): 4894afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise ValueError('Invalid character in operation string') 4904afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.operation = operation 4914afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 4926ae02f0988f60b893b47a5059738060ca3773b02xixuan self._init_message(message) 4934afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 4944afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if not fields: 4954afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.fields = {} 4964afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski else: 4974afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.fields = fields.copy() 4984afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski for key, value in self.fields.iteritems(): 499d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li if type(value) is int: 500d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li value = str(value) 501861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li if self.BAD_CHAR_REGEX.search(key + value): 5024afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise ValueError('Invalid character in %r=%r field' 5034afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski % (key, value)) 5044afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5054afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # build up the timestamp 5064afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if timestamp is None: 5074afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski timestamp = int(time.time()) 5082a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.fields[self.TIMESTAMP_FIELD] = str(timestamp) 5092a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.fields[self.LOCALTIME_FIELD] = time.strftime( 5102a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski '%b %d %H:%M:%S', time.localtime(timestamp)) 5114afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5124afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5134afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def is_start(self): 5144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Indicates if this status log is the start of a new nested block. 5154afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5164afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A boolean indicating if this entry starts a new nested block. 5174afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 5184afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski return self.status_code == 'START' 5194afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5204afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def is_end(self): 5224afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Indicates if this status log is the end of a nested block. 5234afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5244afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A boolean indicating if this entry ends a nested block. 5254afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 5264afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski return self.status_code.startswith('END ') 5274afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5284afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5294afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def render(self): 5304afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Render the status log entry into a text string. 5314afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5324afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A text string suitable for writing into a status log file. 5334afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 5344afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # combine all the log line data into a tab-delimited string 5352a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski subdir = self.subdir or self.RENDERED_NONE_VALUE 5362a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski operation = self.operation or self.RENDERED_NONE_VALUE 5374afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski extra_fields = ['%s=%s' % field for field in self.fields.iteritems()] 5384afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski line_items = [self.status_code, subdir, operation] 5394afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski line_items += extra_fields + [self.message] 5404afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski first_line = '\t'.join(line_items) 5414afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5424afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # append the extra unparsable lines, two-space indented 5434afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski all_lines = [first_line] 5444afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski all_lines += [' ' + line for line in self.extra_message_lines] 5454afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski return '\n'.join(all_lines) 5464afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5474afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5482a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @classmethod 5492a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski def parse(cls, line): 5502a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski """Parse a status log entry from a text string. 5512a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 5522a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski This method is the inverse of render; it should always be true that 5532a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski parse(entry.render()) produces a new status_log_entry equivalent to 5542a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski entry. 5552a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 5562a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @return: A new status_log_entry instance with fields extracted from the 5572a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski given status line. If the line is an extra message line then None 5582a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski is returned. 5592a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski """ 5602a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski # extra message lines are always prepended with two spaces 5612a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if line.startswith(' '): 5622a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski return None 5632a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 5642a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski line = line.lstrip('\t') # ignore indentation 5652a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski entry_parts = line.split('\t') 5662a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if len(entry_parts) < 4: 5672a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski raise ValueError('%r is not a valid status line' % line) 5682a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski status_code, subdir, operation = entry_parts[:3] 5692a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if subdir == cls.RENDERED_NONE_VALUE: 5702a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski subdir = None 5712a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if operation == cls.RENDERED_NONE_VALUE: 5722a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski operation = None 5732a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski message = entry_parts[-1] 5742a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski fields = dict(part.split('=', 1) for part in entry_parts[3:-1]) 5752a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if cls.TIMESTAMP_FIELD in fields: 5762a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski timestamp = int(fields[cls.TIMESTAMP_FIELD]) 5772a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski else: 5782a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski timestamp = None 5792a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski return cls(status_code, subdir, operation, message, fields, timestamp) 5802a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 5812a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 5824afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanskiclass status_indenter(object): 5834afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Abstract interface that a status log indenter should use.""" 5844afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5854afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property 5864afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def indent(self): 5874afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise NotImplementedError 5884afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5894afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5904afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def increment(self): 5914afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Increase indentation by one level.""" 5924afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski raise NotImplementedError 5934afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5944afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5954afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def decrement(self): 5964afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Decrease indentation by one level.""" 5974afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5984afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 5994afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanskiclass status_logger(object): 6004afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Represents a status log file. Responsible for translating messages 6014afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski into on-disk status log lines. 6024afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6034afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property global_filename: The filename to write top-level logs to. 6044afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property subdir_filename: The filename to write subdir-level logs to. 6054afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 6064afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def __init__(self, job, indenter, global_filename='status', 607f0c8224364e104aabe3ff76525e0cc7a39be2a6cAviv Keshet subdir_filename='status', record_hook=None): 6084afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Construct a logger instance. 6094afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6104afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param job: A reference to the job object this is logging for. Only a 6114afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski weak reference to the job is held, to avoid a 6124afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski status_logger <-> job circular reference. 6134afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param indenter: A status_indenter instance, for tracking the 6144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski indentation level. 6154afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param global_filename: An optional filename to initialize the 6164afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.global_filename attribute. 6174afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param subdir_filename: An optional filename to initialize the 6184afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.subdir_filename attribute. 6192a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @param record_hook: An optional function to be called before an entry 6204afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski is logged. The function should expect a single parameter, a 6214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski copy of the status_log_entry object. 6224afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 6234afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self._jobref = weakref.ref(job) 6244afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self._indenter = indenter 6254afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.global_filename = global_filename 6264afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.subdir_filename = subdir_filename 6274afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self._record_hook = record_hook 6284afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6294afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6304afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def render_entry(self, log_entry): 6314afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Render a status_log_entry as it would be written to a log file. 6324afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6334afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param log_entry: A status_log_entry instance to be rendered. 6344afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6354afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: The status log entry, rendered as it would be written to the 6364afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski logs (including indentation). 6374afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 6384afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if log_entry.is_end(): 6394afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski indent = self._indenter.indent - 1 6404afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski else: 6414afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski indent = self._indenter.indent 642bbb026c9fbd2d3907205f4b66232a849d8a424b3jadmanski return '\t' * indent + log_entry.render().rstrip('\n') 6434afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6444afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6452a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski def record_entry(self, log_entry, log_in_subdir=True): 6464afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Record a status_log_entry into the appropriate status log files. 6474afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6484afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param log_entry: A status_log_entry instance to be recorded into the 6494afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski status logs. 6502a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @param log_in_subdir: A boolean that indicates (when true) that subdir 6512a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski logs should be written into the subdirectory status log file. 6524afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 6534afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # acquire a strong reference for the duration of the method 6544afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski job = self._jobref() 6554afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if job is None: 6564afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski logging.warning('Something attempted to write a status log entry ' 6574afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 'after its job terminated, ignoring the attempt.') 6584afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski logging.warning(traceback.format_stack()) 6594afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski return 6604afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6612a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski # call the record hook if one was given 6622a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if self._record_hook: 6632a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self._record_hook(log_entry) 6642a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 6654afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # figure out where we need to log to 6664afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski log_files = [os.path.join(job.resultdir, self.global_filename)] 6672a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if log_in_subdir and log_entry.subdir: 6684afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski log_files.append(os.path.join(job.resultdir, log_entry.subdir, 6694afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self.subdir_filename)) 6704afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6714afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # write out to entry to the log files 6724afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski log_text = self.render_entry(log_entry) 6734afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski for log_file in log_files: 6744afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski fileobj = open(log_file, 'a') 6754afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski try: 6764afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski print >> fileobj, log_text 6774afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski finally: 6784afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski fileobj.close() 6794afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6804afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski # adjust the indentation if this was a START or END entry 6814afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski if log_entry.is_start(): 6824afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self._indenter.increment() 6834afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski elif log_entry.is_end(): 6844afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski self._indenter.decrement() 6854afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6864afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 687da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanskiclass base_job(object): 688da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """An abstract base class for the various autotest job classes. 689da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 6904afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property autodir: The top level autotest directory. 6914afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property clientdir: The autotest client directory. 6924afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property serverdir: The autotest server directory. [OPTIONAL] 6934afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property resultdir: The directory where results should be written out. 6944afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski [WRITABLE] 6954afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 6964afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property pkgdir: The job packages directory. [WRITABLE] 6974afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property tmpdir: The job temporary directory. [WRITABLE] 6984afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property testdir: The job test directory. [WRITABLE] 6994afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property site_testdir: The job site test directory. [WRITABLE] 7004afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7014afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property bindir: The client bin/ directory. 7024afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property profdir: The client profilers/ directory. 7034afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property toolsdir: The client tools/ directory. 7044afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7054afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property control: A path to the control file to be executed. [OPTIONAL] 7064afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property hosts: A set of all live Host objects currently in use by the 7074afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski job. Code running in the context of a local client can safely assume 7084afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski that this set contains only a single entry. 7094afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property machines: A list of the machine names associated with the job. 7104afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property user: The user executing the job. 7114afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property tag: A tag identifying the job. Often used by the scheduler to 7124afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski give a name of the form NUMBER-USERNAME/HOSTNAME. 71391493c88a1506729a6728660d90cfae591cd52aaScott Zawalski @property test_retry: The number of times to retry a test if the test did 71491493c88a1506729a6728660d90cfae591cd52aaScott Zawalski not complete successfully. 7154afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property args: A list of addtional miscellaneous command-line arguments 7164afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski provided when starting the job. 7174afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7184afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property automatic_test_tag: A string which, if set, will be automatically 7194afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski added to the test name when running tests. 7204afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property default_profile_only: A boolean indicating the default value of 7224afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski profile_only used by test.execute. [PERSISTENT] 7234afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property drop_caches: A boolean indicating if caches should be dropped 7244afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski before each test is executed. 7254afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property drop_caches_between_iterations: A boolean indicating if caches 7264afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski should be dropped before each test iteration is executed. 7274afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property run_test_cleanup: A boolean indicating if test.cleanup should be 7284afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski run by default after a test completes, if the run_cleanup argument is 7294afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski not specified. [PERSISTENT] 7304afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7314afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property num_tests_run: The number of tests run during the job. [OPTIONAL] 7324afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property num_tests_failed: The number of tests failed during the job. 7334afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski [OPTIONAL] 7344afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7354afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property harness: An instance of the client test harness. Only available 7364afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski in contexts where client test execution happens. [OPTIONAL] 7374afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property logging: An instance of the logging manager associated with the 7384afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski job. 7394afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property profilers: An instance of the profiler manager associated with 7404afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski the job. 7414afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property sysinfo: An instance of the sysinfo object. Only available in 7424afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski contexts where it's possible to collect sysinfo. 7434afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property warning_manager: A class for managing which types of WARN 7444afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski messages should be logged and which should be supressed. [OPTIONAL] 7454afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @property warning_loggers: A set of readable streams that will be monitored 7464afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski for WARN messages to be logged. [OPTIONAL] 747f53d1269ad7321116eadcb0167d81dbea9bf4befDan Shi @property max_result_size_KB: Maximum size of test results should be 748f53d1269ad7321116eadcb0167d81dbea9bf4befDan Shi collected in KB. [OPTIONAL] 749da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 750da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Abstract methods: 751da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski _find_base_directories [CLASSMETHOD] 752da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Returns the location of autodir, clientdir and serverdir 753da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 754da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski _find_resultdir 755da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Returns the location of resultdir. Gets a copy of any parameters 756da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski passed into base_job.__init__. Can return None to indicate that 757da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski no resultdir is to be used. 7584afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 7594afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski _get_status_logger 7604afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski Returns a status_logger instance for recording job status logs. 761da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 762da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 763f53d1269ad7321116eadcb0167d81dbea9bf4befDan Shi # capture the dependency on several helper classes with factories 764fc3da5bbdcd63840cd224282812c050c5520db11mbligh _job_directory = job_directory 765fc3da5bbdcd63840cd224282812c050c5520db11mbligh _job_state = job_state 766fc3da5bbdcd63840cd224282812c050c5520db11mbligh 767fc3da5bbdcd63840cd224282812c050c5520db11mbligh 768da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # all the job directory attributes 769fc3da5bbdcd63840cd224282812c050c5520db11mbligh autodir = _job_directory.property_factory('autodir') 770fc3da5bbdcd63840cd224282812c050c5520db11mbligh clientdir = _job_directory.property_factory('clientdir') 771fc3da5bbdcd63840cd224282812c050c5520db11mbligh serverdir = _job_directory.property_factory('serverdir') 772fc3da5bbdcd63840cd224282812c050c5520db11mbligh resultdir = _job_directory.property_factory('resultdir') 773fc3da5bbdcd63840cd224282812c050c5520db11mbligh pkgdir = _job_directory.property_factory('pkgdir') 774fc3da5bbdcd63840cd224282812c050c5520db11mbligh tmpdir = _job_directory.property_factory('tmpdir') 775fc3da5bbdcd63840cd224282812c050c5520db11mbligh testdir = _job_directory.property_factory('testdir') 776fc3da5bbdcd63840cd224282812c050c5520db11mbligh site_testdir = _job_directory.property_factory('site_testdir') 777fc3da5bbdcd63840cd224282812c050c5520db11mbligh bindir = _job_directory.property_factory('bindir') 778fc3da5bbdcd63840cd224282812c050c5520db11mbligh profdir = _job_directory.property_factory('profdir') 779fc3da5bbdcd63840cd224282812c050c5520db11mbligh toolsdir = _job_directory.property_factory('toolsdir') 780fc3da5bbdcd63840cd224282812c050c5520db11mbligh 781fc3da5bbdcd63840cd224282812c050c5520db11mbligh 782fc3da5bbdcd63840cd224282812c050c5520db11mbligh # all the generic persistent properties 7839de6ed7ab19c29b5072d52439f434453df16bec0mbligh tag = _job_state.property_factory('_state', 'tag', '') 78491493c88a1506729a6728660d90cfae591cd52aaScott Zawalski test_retry = _job_state.property_factory('_state', 'test_retry', 0) 785fc3da5bbdcd63840cd224282812c050c5520db11mbligh default_profile_only = _job_state.property_factory( 786fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh '_state', 'default_profile_only', False) 787fc3da5bbdcd63840cd224282812c050c5520db11mbligh run_test_cleanup = _job_state.property_factory( 788fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh '_state', 'run_test_cleanup', True) 789fc3da5bbdcd63840cd224282812c050c5520db11mbligh automatic_test_tag = _job_state.property_factory( 790fc3da5bbdcd63840cd224282812c050c5520db11mbligh '_state', 'automatic_test_tag', None) 791f53d1269ad7321116eadcb0167d81dbea9bf4befDan Shi max_result_size_KB = _job_state.property_factory( 792f53d1269ad7321116eadcb0167d81dbea9bf4befDan Shi '_state', 'max_result_size_KB', 0) 793cd36ae0a377f228c69f58d7c5c854ee1a7abf94bXixuan Wu fast = _job_state.property_factory( 794cd36ae0a377f228c69f58d7c5c854ee1a7abf94bXixuan Wu '_state', 'fast', False) 795fc3da5bbdcd63840cd224282812c050c5520db11mbligh 796fc3da5bbdcd63840cd224282812c050c5520db11mbligh # the use_sequence_number property 797fc3da5bbdcd63840cd224282812c050c5520db11mbligh _sequence_number = _job_state.property_factory( 798fc3da5bbdcd63840cd224282812c050c5520db11mbligh '_state', '_sequence_number', None) 799fc3da5bbdcd63840cd224282812c050c5520db11mbligh def _get_use_sequence_number(self): 800fc3da5bbdcd63840cd224282812c050c5520db11mbligh return bool(self._sequence_number) 801fc3da5bbdcd63840cd224282812c050c5520db11mbligh def _set_use_sequence_number(self, value): 802fc3da5bbdcd63840cd224282812c050c5520db11mbligh if value: 803fc3da5bbdcd63840cd224282812c050c5520db11mbligh self._sequence_number = 1 804fc3da5bbdcd63840cd224282812c050c5520db11mbligh else: 805fc3da5bbdcd63840cd224282812c050c5520db11mbligh self._sequence_number = None 806fc3da5bbdcd63840cd224282812c050c5520db11mbligh use_sequence_number = property(_get_use_sequence_number, 807fc3da5bbdcd63840cd224282812c050c5520db11mbligh _set_use_sequence_number) 808da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 80970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi # parent job id is passed in from autoserv command line. It's only used in 81070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi # server job. The property is added here for unittest 81170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi # (base_job_unittest.py) to be consistent on validating public properties of 81270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi # a base_job object. 81370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi parent_job_id = None 814da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 815da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, *args, **dargs): 816da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # initialize the base directories, all others are relative to these 817da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski autodir, clientdir, serverdir = self._find_base_directories() 818da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._autodir = self._job_directory(autodir) 819da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._clientdir = self._job_directory(clientdir) 82091493c88a1506729a6728660d90cfae591cd52aaScott Zawalski # TODO(scottz): crosbug.com/38259, needed to pass unittests for now. 82191493c88a1506729a6728660d90cfae591cd52aaScott Zawalski self.label = None 822da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if serverdir: 823da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._serverdir = self._job_directory(serverdir) 824da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 825da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._serverdir = None 826da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 827da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # initialize all the other directories relative to the base ones 828da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._initialize_dir_properties() 829da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = self._job_directory( 830da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._find_resultdir(*args, **dargs), True) 831da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._execution_contexts = [] 832da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 833fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # initialize all the job state 834fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state = self._job_state() 835fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 836da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 837da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @classmethod 838da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _find_base_directories(cls): 839da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise NotImplementedError() 840da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 841da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 842da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _initialize_dir_properties(self): 843da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 844da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Initializes all the secondary self.*dir properties. Requires autodir, 845da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski clientdir and serverdir to already be initialized. 846da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 847da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # create some stubs for use as shortcuts 848da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def readonly_dir(*args): 849da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return self._job_directory(os.path.join(*args)) 850da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def readwrite_dir(*args): 851da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return self._job_directory(os.path.join(*args), True) 852da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 853da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # various client-specific directories 854da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._bindir = readonly_dir(self.clientdir, 'bin') 855da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._profdir = readonly_dir(self.clientdir, 'profilers') 856da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._pkgdir = readwrite_dir(self.clientdir, 'packages') 857da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._toolsdir = readonly_dir(self.clientdir, 'tools') 858da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 859da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # directories which are in serverdir on a server, clientdir on a client 860d9a056fe72b1a6aa2ccf9acbc3a1918dbbe07028Fang Deng # tmp tests, and site_tests need to be read_write for client, but only 861d9a056fe72b1a6aa2ccf9acbc3a1918dbbe07028Fang Deng # read for server. 862da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if self.serverdir: 863da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski root = self.serverdir 86436bf74a318bf182573ee8d7e0a389293f172c3ffAviv Keshet r_or_rw_dir = readonly_dir 865da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 866da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski root = self.clientdir 86736bf74a318bf182573ee8d7e0a389293f172c3ffAviv Keshet r_or_rw_dir = readwrite_dir 86836bf74a318bf182573ee8d7e0a389293f172c3ffAviv Keshet self._testdir = r_or_rw_dir(root, 'tests') 86936bf74a318bf182573ee8d7e0a389293f172c3ffAviv Keshet self._site_testdir = r_or_rw_dir(root, 'site_tests') 870da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 871da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # various server-specific directories 872da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if self.serverdir: 873d9a056fe72b1a6aa2ccf9acbc3a1918dbbe07028Fang Deng self._tmpdir = readwrite_dir(tempfile.gettempdir()) 874da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 875d9a056fe72b1a6aa2ccf9acbc3a1918dbbe07028Fang Deng self._tmpdir = readwrite_dir(root, 'tmp') 876da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 877da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 878da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _find_resultdir(self, *args, **dargs): 879da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise NotImplementedError() 880da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 881da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 882da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def push_execution_context(self, resultdir): 883da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 884da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Save off the current context of the job and change to the given one. 885da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 886da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski In practice method just changes the resultdir, but it may become more 887da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski extensive in the future. The expected use case is for when a child 888da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski job needs to be executed in some sort of nested context (for example 889da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the way parallel_simple does). The original context can be restored 890da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski with a pop_execution_context call. 891da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 8924afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param resultdir: The new resultdir, relative to the current one. 893da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 894da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski new_dir = self._job_directory( 895da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski os.path.join(self.resultdir, resultdir), True) 896da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._execution_contexts.append(self._resultdir) 897da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = new_dir 898da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 899da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 900da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def pop_execution_context(self): 901da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 902da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Reverse the effects of the previous push_execution_context call. 903da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 9044afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise IndexError: raised when the stack of contexts is empty. 905da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 906da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if not self._execution_contexts: 907da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise IndexError('No old execution context to restore') 908da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = self._execution_contexts.pop() 909fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 910fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 911fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def get_state(self, name, default=_job_state.NO_DEFAULT): 912fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Returns the value associated with a particular name. 913fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 9144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name the value was saved with. 9154afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param default: A default value to return if no state is currently 916fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh associated with var. 917fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 9184afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A deep copy of the value associated with name. Note that this 919fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh explicitly returns a deep copy to avoid problems with mutable 920fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh values; mutations are not persisted or shared. 9214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise KeyError: raised when no state is associated with var and a 922fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh default value is not provided. 923fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 924fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh try: 925fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return self._state.get('public', name, default=default) 926fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh except KeyError: 927fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh raise KeyError(name) 928fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 929fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 930fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set_state(self, name, value): 931fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Saves the value given with the provided name. 932fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 9334afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param name: The name the value should be saved with. 9344afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param value: The value to save. 935fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 936fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state.set('public', name, value) 937fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 938fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 939fc3da5bbdcd63840cd224282812c050c5520db11mbligh def _build_tagged_test_name(self, testname, dargs): 940fc3da5bbdcd63840cd224282812c050c5520db11mbligh """Builds the fully tagged testname and subdirectory for job.run_test. 941fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 9424afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param testname: The base name of the test 9434afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param dargs: The ** arguments passed to run_test. And arguments 944fc3da5bbdcd63840cd224282812c050c5520db11mbligh consumed by this method will be removed from the dictionary. 945fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 9464afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A 3-tuple of the full name of the test, the subdirectory it 947fc3da5bbdcd63840cd224282812c050c5520db11mbligh should be stored in, and the full tag of the subdir. 948fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 949fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag_parts = [] 950fc3da5bbdcd63840cd224282812c050c5520db11mbligh 951fc3da5bbdcd63840cd224282812c050c5520db11mbligh # build up the parts of the tag used for the test name 95274a314b490ff542c4dd2ae4aa0d11c6394d92960Dale Curtis master_testpath = dargs.get('master_testpath', "") 953fc3da5bbdcd63840cd224282812c050c5520db11mbligh base_tag = dargs.pop('tag', None) 954fc3da5bbdcd63840cd224282812c050c5520db11mbligh if base_tag: 955fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag_parts.append(str(base_tag)) 956fc3da5bbdcd63840cd224282812c050c5520db11mbligh if self.use_sequence_number: 957fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag_parts.append('_%02d_' % self._sequence_number) 958fc3da5bbdcd63840cd224282812c050c5520db11mbligh self._sequence_number += 1 959fc3da5bbdcd63840cd224282812c050c5520db11mbligh if self.automatic_test_tag: 960fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag_parts.append(self.automatic_test_tag) 961fc3da5bbdcd63840cd224282812c050c5520db11mbligh full_testname = '.'.join([testname] + tag_parts) 962fc3da5bbdcd63840cd224282812c050c5520db11mbligh 963fc3da5bbdcd63840cd224282812c050c5520db11mbligh # build up the subdir and tag as well 964fc3da5bbdcd63840cd224282812c050c5520db11mbligh subdir_tag = dargs.pop('subdir_tag', None) 965fc3da5bbdcd63840cd224282812c050c5520db11mbligh if subdir_tag: 966fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag_parts.append(subdir_tag) 967fc3da5bbdcd63840cd224282812c050c5520db11mbligh subdir = '.'.join([testname] + tag_parts) 96874a314b490ff542c4dd2ae4aa0d11c6394d92960Dale Curtis subdir = os.path.join(master_testpath, subdir) 969fc3da5bbdcd63840cd224282812c050c5520db11mbligh tag = '.'.join(tag_parts) 970fc3da5bbdcd63840cd224282812c050c5520db11mbligh 971fc3da5bbdcd63840cd224282812c050c5520db11mbligh return full_testname, subdir, tag 972fc3da5bbdcd63840cd224282812c050c5520db11mbligh 973fc3da5bbdcd63840cd224282812c050c5520db11mbligh 974fc3da5bbdcd63840cd224282812c050c5520db11mbligh def _make_test_outputdir(self, subdir): 975fc3da5bbdcd63840cd224282812c050c5520db11mbligh """Creates an output directory for a test to run it. 976fc3da5bbdcd63840cd224282812c050c5520db11mbligh 9774afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param subdir: The subdirectory of the test. Generally computed by 978fc3da5bbdcd63840cd224282812c050c5520db11mbligh _build_tagged_test_name. 979fc3da5bbdcd63840cd224282812c050c5520db11mbligh 9804afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @return: A job_directory instance corresponding to the outputdir of 981fc3da5bbdcd63840cd224282812c050c5520db11mbligh the test. 9824afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @raise TestError: If the output directory is invalid. 983fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 984fc3da5bbdcd63840cd224282812c050c5520db11mbligh # explicitly check that this subdirectory is new 985fc3da5bbdcd63840cd224282812c050c5520db11mbligh path = os.path.join(self.resultdir, subdir) 986fc3da5bbdcd63840cd224282812c050c5520db11mbligh if os.path.exists(path): 987fc3da5bbdcd63840cd224282812c050c5520db11mbligh msg = ('%s already exists; multiple tests cannot run with the ' 988fc3da5bbdcd63840cd224282812c050c5520db11mbligh 'same subdirectory' % subdir) 989fc3da5bbdcd63840cd224282812c050c5520db11mbligh raise error.TestError(msg) 990fc3da5bbdcd63840cd224282812c050c5520db11mbligh 991fc3da5bbdcd63840cd224282812c050c5520db11mbligh # create the outputdir and raise a TestError if it isn't valid 992fc3da5bbdcd63840cd224282812c050c5520db11mbligh try: 993fc3da5bbdcd63840cd224282812c050c5520db11mbligh outputdir = self._job_directory(path, True) 994fc3da5bbdcd63840cd224282812c050c5520db11mbligh return outputdir 995fc3da5bbdcd63840cd224282812c050c5520db11mbligh except self._job_directory.JobDirectoryException, e: 996fc3da5bbdcd63840cd224282812c050c5520db11mbligh logging.exception('%s directory creation failed with %s', 997fc3da5bbdcd63840cd224282812c050c5520db11mbligh subdir, e) 998fc3da5bbdcd63840cd224282812c050c5520db11mbligh raise error.TestError('%s directory creation failed' % subdir) 9994afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 10004afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 10014afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski def record(self, status_code, subdir, operation, status='', 10024afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski optional_fields=None): 10034afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """Record a job-level status event. 10044afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 10054afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski Logs an event noteworthy to the Autotest job as a whole. Messages will 10064afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski be written into a global status log file, as well as a subdir-local 10074afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski status log file (if subdir is specified). 10084afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski 10094afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param status_code: A string status code describing the type of status 10104afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski entry being recorded. It must pass log.is_valid_status to be 10114afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski considered valid. 10124afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param subdir: A specific results subdirectory this also applies to, or 10134afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski None. If not None the subdirectory must exist. 10144afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param operation: A string describing the operation that was run. 10154afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param status: An optional human-readable message describing the status 10164afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski entry, for example an error message or "completed successfully". 10174afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski @param optional_fields: An optional dictionary of addtional named fields 10184afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski to be included with the status message. Every time timestamp and 10194afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski localtime entries are generated with the current time and added 10204afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski to this dictionary. 10214afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski """ 10224afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski entry = status_log_entry(status_code, subdir, operation, status, 10234afc3676e5735bd52303c0612b1cbc62c2d6fba8jadmanski optional_fields) 10242a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.record_entry(entry) 10252a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 10262a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 10272a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski def record_entry(self, entry, log_in_subdir=True): 10282a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski """Record a job-level status event, using a status_log_entry. 10292a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 10302a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski This is the same as self.record but using an existing status log 10312a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski entry object rather than constructing one for you. 10322a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 10332a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @param entry: A status_log_entry object 10342a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski @param log_in_subdir: A boolean that indicates (when true) that subdir 10352a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski logs should be written into the subdirectory status log file. 10362a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski """ 10372a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self._get_status_logger().record_entry(entry, log_in_subdir) 1038