base_job.py revision fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4
1fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mblighimport os, copy, logging, errno, tempfile, cPickle as pickle 2da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 3da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanskifrom autotest_lib.client.common_lib import autotemp, error 4da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 5da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 6da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanskiclass job_directory(object): 7da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Represents a job.*dir directory.""" 8da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 9da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 10da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski class MissingDirectoryException(error.AutotestError): 11da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a directory required by the job does not exist.""" 12da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path): 13da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, 'Directory %s does not exist' % path) 14da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 15da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 16da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski class UncreatableDirectoryException(error.AutotestError): 17da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a directory required by the job is missing and cannot 18da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski be created.""" 19da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path, error): 20da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg = 'Creation of directory %s failed with exception %s' 21da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg %= (path, error) 22da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, msg) 23da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 24da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 25da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski class UnwritableDirectoryException(error.AutotestError): 26da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """Raised when a writable directory required by the job exists 27da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski but is not writable.""" 28da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path): 29da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski msg = 'Directory %s exists but is not writable' % path 30da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Exception.__init__(self, msg) 31da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 32da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 33da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, path, is_writable=False): 34da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 35da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Instantiate a job directory. 36da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 37da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param path The path of the directory. If None a temporary directory 38da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski will be created instead. 39da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param is_writable If True, expect the directory to be writable. 40da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 41da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises MissingDirectoryException raised if is_writable=False and the 42da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist. 43da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UnwritableDirectoryException raised if is_writable=True and 44da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the directory exists but is not writable. 45da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UncreatableDirectoryException raised if is_writable=True, the 46da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist and it cannot be created. 47da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 48da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if path is None: 49da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if is_writable: 50da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._tempdir = autotemp.tempdir(unique_id='autotest') 51da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self.path = self._tempdir.name 52da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 53da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.MissingDirectoryException(path) 54da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 55da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._tempdir = None 56da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self.path = path 57da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._ensure_valid(is_writable) 58da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 59da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 60da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _ensure_valid(self, is_writable): 61da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 62da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Ensure that this is a valid directory. 63da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 64da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Will check if a directory exists, can optionally also enforce that 65da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski it be writable. It can optionally create it if necessary. Creation 66da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski will still fail if the path is rooted in a non-writable directory, or 67da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if a file already exists at the given location. 68da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 69da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param dir_path A path where a directory should be located 70da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param is_writable A boolean indicating that the directory should 71da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski not only exist, but also be writable. 72da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 73da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises MissingDirectoryException raised if is_writable=False and the 74da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist. 75da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UnwritableDirectoryException raised if is_writable=True and 76da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the directory is not wrtiable. 77da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises UncreatableDirectoryException raised if is_writable=True, the 78da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski directory does not exist and it cannot be created 79da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 808054b0da315c94d249aaf80512bc133d4cb55b92mbligh # ensure the directory exists 818054b0da315c94d249aaf80512bc133d4cb55b92mbligh if is_writable: 828054b0da315c94d249aaf80512bc133d4cb55b92mbligh try: 838054b0da315c94d249aaf80512bc133d4cb55b92mbligh os.makedirs(self.path) 848054b0da315c94d249aaf80512bc133d4cb55b92mbligh except OSError, e: 85fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if e.errno != errno.EEXIST or not os.path.isdir(self.path): 86da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.UncreatableDirectoryException(self.path, e) 878054b0da315c94d249aaf80512bc133d4cb55b92mbligh elif not os.path.isdir(self.path): 888054b0da315c94d249aaf80512bc133d4cb55b92mbligh raise self.MissingDirectoryException(self.path) 89da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 90da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # if is_writable=True, also check that the directory is writable 91da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if is_writable and not os.access(self.path, os.W_OK): 92da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise self.UnwritableDirectoryException(self.path) 93da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 94da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 95da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @staticmethod 96da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def property_factory(attribute): 97da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 98da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Create a job.*dir -> job._*dir.path property accessor. 99da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 100da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param attribute A string with the name of the attribute this is 101da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski exposed as. '_'+attribute must then be attribute that holds 102da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski either None or a job_directory-like object. 103da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 104da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @returns A read-only property object that exposes a job_directory path 105da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 106da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @property 107da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def dir_property(self): 108da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski underlying_attribute = getattr(self, '_' + attribute) 109da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if underlying_attribute is None: 110da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return None 111da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 112da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return underlying_attribute.path 113da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return dir_property 114da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 115da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 116fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mblighclass job_state(object): 117fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """A class for managing explicit job and user state, optionally persistent. 118fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 119fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh The class allows you to save state by name (like a dictionary). Any state 120fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh stored in this class should be picklable and deep copyable. While this is 121fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh not enforced it is recommended that only valid python identifiers be used 122fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh as names. Additionally, the namespace 'stateful_property' is used for 123fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh storing the valued associated with properties constructed using the 124fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh property_factory method. 125fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 126fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 127fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh NO_DEFAULT = object() 128fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh PICKLE_PROTOCOL = 2 # highest protocol available in python 2.4 129fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 130fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 131fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def __init__(self): 132fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Initialize the job state.""" 133fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state = {} 134fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._backing_file = None 135fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 136fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 137fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def get(self, namespace, name, default=NO_DEFAULT): 138fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Returns the value associated with a particular name. 139fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 140fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param namespace The namespace that the property should be stored in. 141fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name the value was saved with. 142fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param default A default value to return if no state is currently 143fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh associated with var. 144fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 145fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns A deep copy of the value associated with name. Note that this 146fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh explicitly returns a deep copy to avoid problems with mutable 147fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh values; mutations are not persisted or shared. 148fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @raises KeyError raised when no state is associated with var and a 149fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh default value is not provided. 150fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 151fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if self.has(namespace, name): 152fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return copy.deepcopy(self._state[namespace][name]) 153fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh elif default is self.NO_DEFAULT: 154fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh raise KeyError('No key %s in namespace %s' % (name, namespace)) 155fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 156fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return default 157fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 158fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 159fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def read_from_file(self, file_path): 160fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Read in any state from the file at file_path. 161fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 162fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh Any state specified only in-memory will be preserved. Any state 163fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh specified on-disk will be set in-memory, even if an in-memory 164fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh setting already exists. In the special case that the file does 165fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh not exist it is treated as empty and not a failure. 166fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 167fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # if the file exists, pull out its contents 168fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if file_path: 169fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh try: 170fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh on_disk_state = pickle.load(open(file_path)) 171fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh except IOError, e: 172fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if e.errno == errno.ENOENT: 173fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.info('Persistent state file %s does not exist', 174fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh file_path) 175fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return 176fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 177fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh raise 178fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 179fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return 180fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 181fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # merge the on-disk state with the in-memory state 182fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh for namespace, namespace_dict in on_disk_state.iteritems(): 183fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh in_memory_namespace = self._state.setdefault(namespace, {}) 184fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh for name, value in namespace_dict.iteritems(): 185fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if name in in_memory_namespace: 186fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if in_memory_namespace[name] != value: 187fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.info('Persistent value of %s.%s from %s ' 188fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 'overridding existing in-memory value', 189fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace, name, file_path) 190fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh in_memory_namespace[name] = value 191fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 192fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Value of %s.%s is unchanged, skipping' 193fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 'import', namespace, name) 194fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 195fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Importing %s.%s from state file %s', 196fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace, name, file_path) 197fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh in_memory_namespace[name] = value 198fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 199fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # flush the merged state out to disk 200fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._write_to_backing_file() 201fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 202fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 203fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def write_to_file(self, file_path): 204fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Write out the current state to the given path. 205fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 206fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param file_path The path where the state should be written out to. 207fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh Must be writable. 208fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 209fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh outfile = open(file_path, 'w') 210fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh try: 211fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh pickle.dump(self._state, outfile, self.PICKLE_PROTOCOL) 212fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh finally: 213fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh outfile.close() 214fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Persistent state flushed to %s', file_path) 215fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 216fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 217fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def _write_to_backing_file(self): 218fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Flush the current state to the backing file.""" 219fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if self._backing_file: 220fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self.write_to_file(self._backing_file) 221fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 222fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 223fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set_backing_file(self, file_path): 224fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Change the path used as the backing file for the persistent state. 225fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 226fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh When a new backing file is specified if a file already exists then 227fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh its contents will be added into the current state, with conflicts 228fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh between the file and memory being resolved in favor of the file 229fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh contents. The file will then be kept in sync with the (combined) 230fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh in-memory state. The syncing can be disabled by setting this to None. 231fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 232fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param file_path A path on the filesystem that can be read from and 233fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh written to, or None to turn off the backing store. 234fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 235fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._backing_file = None 236fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self.read_from_file(file_path) 237fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._backing_file = file_path 238fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._write_to_backing_file() 239fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 240fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 241fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set(self, namespace, name, value): 242fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Saves the value given with the provided name. 243fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 244fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param namespace The namespace that the property should be stored in. 245fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name the value should be saved with. 246fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param value The value to save. 247fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 248fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace_dict = self._state.setdefault(namespace, {}) 249fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace_dict[name] = copy.deepcopy(value) 250fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._write_to_backing_file() 251fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Persistent state %s.%s now set to %r', namespace, 252fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh name, value) 253fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 254fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 255fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def has(self, namespace, name): 256fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Return a boolean indicating if namespace.name is defined. 257fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 258fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param namespace The namespace to check for a definition. 259fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name to check for a definition. 260fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 261fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns True if the given name is defined in the given namespace and 262fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh False otherwise. 263fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 264fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return namespace in self._state and name in self._state[namespace] 265fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 266fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 267fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def discard(self, namespace, name): 268fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """If namespace.name is a defined value, deletes it. 269fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 270fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param namespace The namespace that the property is stored in. 271fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name the value is saved with. 272fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 273fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if self.has(namespace, name): 274fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh del self._state[namespace][name] 275fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if len(self._state[namespace]) == 0: 276fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh del self._state[namespace] 277fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._write_to_backing_file() 278fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug('Persistent state %s.%s deleted', namespace, name) 279fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh else: 280fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh logging.debug( 281fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 'Persistent state %s.%s not defined so nothing is discarded', 282fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh namespace, name) 283fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 284fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 285fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @staticmethod 286fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def property_factory(state_attribute, property_attribute, default): 287fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 288fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh Create a property object for an attribute using self.get and self.set. 289fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 290fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param state_attribute A string with the name of the attribute on 291fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh job that contains the job_state instance. 292fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param property_attribute A string with the name of the attribute 293fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh this property is exposed as. 294fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param default A default value that should be used for this property 295fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if it is not set. 296fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 297fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns A read-write property object that performs self.get calls 298fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh to read the value and self.set calls to set it. 299fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 300fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def getter(job): 301fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh state = getattr(job, state_attribute) 302fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return state.get('stateful_property', property_attribute, default) 303fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def setter(job, value): 304fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh state = getattr(job, state_attribute) 305fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh state.set('stateful_property', property_attribute, value) 306fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return property(getter, setter) 307fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 308fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 309da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanskiclass base_job(object): 310da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """An abstract base class for the various autotest job classes. 311da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 312da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Properties: 313da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski autodir 314da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The top level autotest directory. 315da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski clientdir 316da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The autotest client directory. 317da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski serverdir 318da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The autotest server directory. [OPTIONAL] 319da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski resultdir 320da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The directory where results should be written out. If not specified 321da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski then results should not be written anywhere. [WRITABLE] 322da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 323da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski pkgdir 324da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The job packages directory. [WRITABLE] 325da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski tmpdir 326da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The job temporary directory. [WRITABLE] 327da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski testdir 328da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The job test directory. [WRITABLE] 329da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski site_testdir 330da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The job site test directory. [WRITABLE] 331da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 332da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski bindir 333da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The client bin/ directory. 334da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski configdir 335da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The client config/ directory. 336da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski profdir 337da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The client profilers/ directory. 338da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski toolsdir 339da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The client tools/ directory. 340da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 341da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski conmuxdir 342da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The conmux directory. [OPTIONAL] 343da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 344da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski control 345da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A path to the control file to be executed. [OPTIONAL] 346da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski hosts 347da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A set of all live Host objects currently in use by the job. 348da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski machines 349da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A list of the machine names associated with the job. 350da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski user 351da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The user executing the job. 352da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski tag 353da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A tag identifying the job. Often used by the scheduler to give 354da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski a name of the form NUMBER-USERNAME/HOSTNAME. [OPTIONAL] 355da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 356da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski last_boot_tag 357fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh The label of the kernel from the last reboot. [OPTIONAL,PERSISTENT] 358da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski default_profile_only 359da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A boolean indicating the default value of profile_only used 360fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh by test.execute. [PERSISTENT] 361da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski drop_caches 362da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A boolean indicating if caches should be dropped before each 363da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski test is executed. 364da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski drop_caches_between_iterations 365da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A boolean indicating if caches should be dropped before each 366da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski test iteration is executed. 367fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh run_test_cleanup 368fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh A boolean indicating if test.cleanup should be run by default 369fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh after a test completes, if the run_cleanup argument is not 370fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh specified. [PERSISTENT] 371da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 372da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski num_tests_run 373da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The number of tests run during the job. [OPTIONAL] 374da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski num_tests_failed 375da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski The number of tests failed during the job. [OPTIONAL] 376da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 377da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski bootloader 378da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski An instance of the boottool class. May not be available on job 379da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski instances where access to the bootloader is not available 380da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski (e.g. on the server running a server job). [OPTIONAL] 381da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski harness 382da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski An instance of the client test harness. Only available in contexts 383da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski where client test execution happens. [OPTIONAL] 384da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski logging 385da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski An instance of the logging manager associated with the job. 386da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski profilers 387da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski An instance of the profiler manager associated with the job. 388da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski sysinfo 389da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski An instance of the sysinfo object. Only available in contexts 390da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski where it's possible to collect sysinfo. 391da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski warning_manager 392da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A class for managing which types of WARN messages should be 393da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski logged and which should be supressed. [OPTIONAL] 394da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski warning_loggers 395da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski A set of readable streams that will be monitored for WARN messages 396da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski to be logged. [OPTIONAL] 397da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 398da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Abstract methods: 399da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski _find_base_directories [CLASSMETHOD] 400da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Returns the location of autodir, clientdir and serverdir 401da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 402da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski _find_resultdir 403da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Returns the location of resultdir. Gets a copy of any parameters 404da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski passed into base_job.__init__. Can return None to indicate that 405da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski no resultdir is to be used. 406da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 407da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 408da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # all the job directory attributes 409da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski autodir = job_directory.property_factory('autodir') 410da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski clientdir = job_directory.property_factory('clientdir') 411da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski serverdir = job_directory.property_factory('serverdir') 412da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski resultdir = job_directory.property_factory('resultdir') 413da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski pkgdir = job_directory.property_factory('pkgdir') 414da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski tmpdir = job_directory.property_factory('tmpdir') 415da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski testdir = job_directory.property_factory('testdir') 416da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski site_testdir = job_directory.property_factory('site_testdir') 417da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski bindir = job_directory.property_factory('bindir') 418da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski configdir = job_directory.property_factory('configdir') 419da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski profdir = job_directory.property_factory('profdir') 420da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski toolsdir = job_directory.property_factory('toolsdir') 421da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski conmuxdir = job_directory.property_factory('conmuxdir') 422da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 423da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 424fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # all the persistent properties 425fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh default_profile_only = job_state.property_factory( 426fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh '_state', 'default_profile_only', False) 427fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh run_test_cleanup = job_state.property_factory( 428fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh '_state', 'run_test_cleanup', True) 429fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh last_boot_tag = job_state.property_factory('_state', 'last_boot_tag', None) 430fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 431fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 432fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # capture the dependency on several helper classes with factories 433da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski _job_directory = job_directory 434fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh _job_state = job_state 435da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 436da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 437da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def __init__(self, *args, **dargs): 438da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # initialize the base directories, all others are relative to these 439da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski autodir, clientdir, serverdir = self._find_base_directories() 440da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._autodir = self._job_directory(autodir) 441da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._clientdir = self._job_directory(clientdir) 442da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if serverdir: 443da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._serverdir = self._job_directory(serverdir) 444da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 445da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._serverdir = None 446da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 447da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # initialize all the other directories relative to the base ones 448da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._initialize_dir_properties() 449da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = self._job_directory( 450da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._find_resultdir(*args, **dargs), True) 451da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._execution_contexts = [] 452da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 453fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh # initialize all the job state 454fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state = self._job_state() 455fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 456da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 457da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @classmethod 458da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _find_base_directories(cls): 459da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise NotImplementedError() 460da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 461da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 462da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _initialize_dir_properties(self): 463da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 464da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Initializes all the secondary self.*dir properties. Requires autodir, 465da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski clientdir and serverdir to already be initialized. 466da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 467da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # create some stubs for use as shortcuts 468da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def readonly_dir(*args): 469da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return self._job_directory(os.path.join(*args)) 470da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def readwrite_dir(*args): 471da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski return self._job_directory(os.path.join(*args), True) 472da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 473da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # various client-specific directories 474da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._bindir = readonly_dir(self.clientdir, 'bin') 475da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._configdir = readonly_dir(self.clientdir, 'config') 476da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._profdir = readonly_dir(self.clientdir, 'profilers') 477da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._pkgdir = readwrite_dir(self.clientdir, 'packages') 478da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._toolsdir = readonly_dir(self.clientdir, 'tools') 479da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 480da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # directories which are in serverdir on a server, clientdir on a client 481da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if self.serverdir: 482da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski root = self.serverdir 483da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 484da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski root = self.clientdir 485da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._tmpdir = readwrite_dir(root, 'tmp') 486da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._testdir = readwrite_dir(root, 'tests') 487da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._site_testdir = readwrite_dir(root, 'site_tests') 488da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 489da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski # various server-specific directories 490da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if self.serverdir: 491da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._conmuxdir = readonly_dir(self.autodir, 'conmux') 492da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski else: 493da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._conmuxdir = None 494da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 495da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 496da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def _find_resultdir(self, *args, **dargs): 497da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise NotImplementedError() 498da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 499da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 500da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def push_execution_context(self, resultdir): 501da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 502da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Save off the current context of the job and change to the given one. 503da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 504da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski In practice method just changes the resultdir, but it may become more 505da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski extensive in the future. The expected use case is for when a child 506da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski job needs to be executed in some sort of nested context (for example 507da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski the way parallel_simple does). The original context can be restored 508da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski with a pop_execution_context call. 509da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 510da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @param resultdir The new resultdir, relative to the current one. 511da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 512da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski new_dir = self._job_directory( 513da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski os.path.join(self.resultdir, resultdir), True) 514da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._execution_contexts.append(self._resultdir) 515da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = new_dir 516da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 517da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 518da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski def pop_execution_context(self): 519da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 520da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski Reverse the effects of the previous push_execution_context call. 521da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski 522da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski @raises IndexError raised when the stack of contexts is empty. 523da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski """ 524da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski if not self._execution_contexts: 525da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski raise IndexError('No old execution context to restore') 526da2f143ff7658eca507041cd2393ead3b68cfa5ejadmanski self._resultdir = self._execution_contexts.pop() 527fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 528fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 529fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def get_state(self, name, default=_job_state.NO_DEFAULT): 530fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Returns the value associated with a particular name. 531fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 532fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name the value was saved with. 533fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param default A default value to return if no state is currently 534fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh associated with var. 535fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 536fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns A deep copy of the value associated with name. Note that this 537fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh explicitly returns a deep copy to avoid problems with mutable 538fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh values; mutations are not persisted or shared. 539fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @raises KeyError raised when no state is associated with var and a 540fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh default value is not provided. 541fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 542fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh try: 543fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return self._state.get('public', name, default=default) 544fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh except KeyError: 545fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh raise KeyError(name) 546fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 547fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 548fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def set_state(self, name, value): 549fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Saves the value given with the provided name. 550fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 551fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name the value should be saved with. 552fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param value The value to save. 553fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 554fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state.set('public', name, value) 555fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 556fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 557fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def has_state(self, name): 558fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Returns a boolean indicating if the given name is defined. 559fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 560fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name to check for a definition. 561fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 562fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns True if state is associated with name, False otherwise. 563fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 564fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return self._state.has('public', name) 565fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 566fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 567fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def discard_state(self, name): 568fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Discards the state with the provided name. 569fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 570fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param name The name of the value that should be discarded. 571fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 572fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state.discard('public', name) 573fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 574fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 575fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def save_state(self, file_path=None): 576fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Saves the entire job state into a file. 577fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 578fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param file_path A writable file path that the job state can be written 579fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh into. If None a temporary file is created instead. 580fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 581fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @returns The path of the file the state was written into. If file_path 582fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh is not None then this will always be file_path. 583fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 584fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh if file_path is None: 585fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh fd, file_path = tempfile.mkstemp(dir=self.tmpdir) 586fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh os.close(fd) 587fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state.write_to_file(file_path) 588fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh return file_path 589fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 590fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 591fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh def load_state(self, file_path): 592fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """Loads job state from a file, overriding any in-memory state. 593fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh 594fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh @param file_path The path of a file that contains job state. Should 595fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh always be a file produced by a save_state call. 596fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh """ 597fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh self._state.read_from_file(file_path) 598