1#!/usr/bin/python
2
3"""A script that provides convertion between models.job and a protocol
4buffer object.
5
6This script contains only one class that takes an job instance and
7convert it into a protocol buffer object. The class will also be
8responsible for serializing the job instance via protocol buffers.
9
10"""
11
12# import python libraries
13import datetime
14import time
15
16# import autotest libraries
17from autotest_lib.tko import models
18from autotest_lib.tko import tko_pb2
19from autotest_lib.tko import utils
20
21__author__ = 'darrenkuo@google.com (Darren Kuo)'
22
23mktime = time.mktime
24datetime = datetime.datetime
25
26class JobSerializer(object):
27    """A class that takes a job object of the tko module and package
28    it with a protocol buffer.
29
30    This class will take a model.job object as input and create a
31    protocol buffer to include all the content of the job object. This
32    protocol buffer object will be serialized into a binary file.
33    """
34
35    def __init__(self):
36
37        self.job_type_dict = {'dir':str, 'tests':list, 'user':str,
38                              'label':str, 'machine':str,
39                              'queued_time':datetime,
40                              'started_time':datetime,
41                              'finished_time':datetime,
42                              'machine_owner':str,
43                              'machine_group':str, 'aborted_by':str,
44                              'aborted_on':datetime,
45                              'keyval_dict':dict,
46                              'afe_parent_job_id':str,
47                              'build_version':str,
48                              'suite':str,
49                              'board':str}
50
51        self.test_type_dict = {'subdir':str, 'testname':str,
52                               'status':str, 'reason':str,
53                               'kernel':models.kernel, 'machine':str,
54                               'started_time':datetime,
55                               'finished_time':datetime,
56                               'iterations':list, 'attributes':dict,
57                               'labels':list}
58
59        self.kernel_type_dict = {'base':str, 'kernel_hash':str}
60
61        self.iteration_type_dict = {'index':int, 'attr_keyval':dict,
62                                    'perf_keyval':dict}
63
64
65    def deserialize_from_binary(self, infile):
66        """Takes in a binary file name and returns a tko job object.
67
68        The method first deserialize the binary into a protocol buffer
69        job object and then converts the job object into a tko job
70        object.
71
72
73        @param infile: the name of the binary file that will be deserialized.
74
75        @return: a tko job that is represented by the binary file will
76        be returned.
77        """
78
79        job_pb = tko_pb2.Job()
80
81        binary = open(infile, 'r')
82        try:
83            job_pb.ParseFromString(binary.read())
84        finally:
85            binary.close()
86
87        return self.get_tko_job(job_pb)
88
89
90    def serialize_to_binary(self, the_job, tag, binaryfilename):
91        """Serializes the tko job object into a binary by using a
92        protocol buffer.
93
94        The method takes a tko job object and constructs a protocol
95        buffer job object. Then invokes the native serializing
96        function on the object to get a binary string. The string is
97        then written to outfile.
98
99        Precondition: Assumes that all the information about the job
100        is already in the job object. Any fields that is None will be
101        provided a default value.
102
103        @param the_job: the tko job object that will be serialized.
104        tag: contains the job name and the afe_job_id
105        binaryfilename: the name of the file that will be written to
106        @param tag: The job tag string.
107        @param binaryfilename: The output filename.
108
109        @return: the filename of the file that contains the
110        binary of the serialized object.
111        """
112
113        pb_job = tko_pb2.Job()
114        self.set_pb_job(the_job, pb_job, tag)
115
116        out = open(binaryfilename, 'wb')
117        try:
118            out.write(pb_job.SerializeToString())
119        finally:
120            out.close()
121
122
123    def set_afe_job_id_and_tag(self, pb_job, tag):
124        """Sets the pb job's afe_job_id and tag field.
125
126        @param
127        pb_job: the pb job that will have it's fields set.
128        tag: used to set pb_job.tag and pb_job.afe_job_id.
129        """
130        pb_job.tag = tag
131        pb_job.afe_job_id = utils.get_afe_job_id(tag)
132
133
134    # getter setter methods
135    def get_tko_job(self, job):
136        """Creates a a new tko job object from the pb job object.
137
138        Uses getter methods on the pb objects to extract all the
139        attributes and finally constructs a tko job object using the
140        models.job constructor.
141
142        @param
143        job: a pb job where data is being extracted from.
144
145        @return a tko job object.
146        """
147
148        fields_dict = self.get_trivial_attr(job, self.job_type_dict)
149
150        fields_dict['tests'] = [self.get_tko_test(test) for test in job.tests]
151
152        fields_dict['keyval_dict'] = dict((keyval.name, keyval.value)
153                                          for keyval in job.keyval_dict)
154
155        newjob = models.job(fields_dict['dir'], fields_dict['user'],
156                            fields_dict['label'],
157                            fields_dict['machine'],
158                            fields_dict['queued_time'],
159                            fields_dict['started_time'],
160                            fields_dict['finished_time'],
161                            fields_dict['machine_owner'],
162                            fields_dict['machine_group'],
163                            fields_dict['aborted_by'],
164                            fields_dict['aborted_on'],
165                            fields_dict['keyval_dict'])
166
167        newjob.tests.extend(fields_dict['tests'])
168
169        return newjob
170
171
172    def set_pb_job(self, tko_job, pb_job, tag):
173        """Set the fields for the new job object.
174
175        Method takes in a tko job and an empty protocol buffer job
176        object.  Then safely sets all the appropriate field by first
177        testing if the value in the original object is None.
178
179        @param
180        tko_job: a tko job instance that will have it's values
181        transfered to the new job
182        pb_job: a new instance of the job class provided in the
183        protocol buffer.
184        tag: used to set pb_job.tag and pb_job.afe_job_id.
185        """
186
187        self.set_trivial_attr(tko_job, pb_job, self.job_type_dict)
188        self.set_afe_job_id_and_tag(pb_job, tag)
189        if hasattr(tko_job, 'index'):
190            pb_job.job_idx = tko_job.index
191
192        for test in tko_job.tests:
193            newtest = pb_job.tests.add()
194            self.set_pb_test(test, newtest)
195
196        for key, val in tko_job.keyval_dict.iteritems():
197            newkeyval = pb_job.keyval_dict.add()
198            newkeyval.name = key
199            newkeyval.value = str(val)
200
201
202    def get_tko_test(self, test):
203        """Creates a tko test from pb_test.
204
205        Extracts data from pb_test by calling helper methods and
206        creates a tko test using the models.test constructor.
207
208        @param:
209        test: a pb_test where fields will be extracted from.
210
211        @return a new instance of models.test
212        """
213        fields_dict = self.get_trivial_attr(test, self.test_type_dict)
214
215        fields_dict['kernel'] = self.get_tko_kernel(test.kernel)
216
217        fields_dict['iterations'] = [self.get_tko_iteration(iteration)
218                                     for iteration in test.iterations]
219
220        fields_dict['attributes'] = dict((keyval.name, keyval.value)
221                                         for keyval in test.attributes)
222
223        fields_dict['labels'] = list(test.labels)
224
225        # The constructor for models.test accepts a "perf_values" list that
226        # represents performance values of the test.  The empty list argument
227        # in the constructor call below represents this value and makes this
228        # code adhere properly to the models.test constructor argument list.
229        # However, the effect of the empty list is that perf values are
230        # ignored in the job_serializer module.  This is ok for now because
231        # autotest does not use the current module.  If job_serializer is used
232        # in the future, we need to modify the "tko.proto" protobuf file to
233        # understand the notion of perf_values, then modify this file
234        # accordingly to use it.
235        return models.test(fields_dict['subdir'],
236                           fields_dict['testname'],
237                           fields_dict['status'],
238                           fields_dict['reason'],
239                           fields_dict['kernel'],
240                           fields_dict['machine'],
241                           fields_dict['started_time'],
242                           fields_dict['finished_time'],
243                           fields_dict['iterations'],
244                           fields_dict['attributes'],
245                           [],
246                           fields_dict['labels'])
247
248
249    def set_pb_test(self, tko_test, pb_test):
250        """Sets the various fields of test object of the tko protocol.
251
252        Method takes a tko test and a new test of the protocol buffer and
253        transfers the values in the tko test to the new test.
254
255        @param
256        tko_test: a tko test instance.
257        pb_test: an empty protocol buffer test instance.
258
259        """
260
261        self.set_trivial_attr(tko_test, pb_test, self.test_type_dict)
262
263        self.set_pb_kernel(tko_test.kernel, pb_test.kernel)
264        if hasattr(tko_test, 'test_idx'):
265            pb_test.test_idx = tko_test.test_idx
266
267        for current_iteration in tko_test.iterations:
268            pb_iteration = pb_test.iterations.add()
269            self.set_pb_iteration(current_iteration, pb_iteration)
270
271        for key, val in tko_test.attributes.iteritems():
272            newkeyval = pb_test.attributes.add()
273            newkeyval.name = key
274            newkeyval.value = str(val)
275
276        for current_label in tko_test.labels:
277            pb_test.labels.append(current_label)
278
279
280    def get_tko_kernel(self, kernel):
281        """Constructs a new tko kernel object from a pb kernel object.
282
283        Uses all the getter methods on the pb kernel object to extract
284        the attributes and constructs a new tko kernel object using
285        the model.kernel constructor.
286
287        @param
288        kernel: a pb kernel object where data will be extracted.
289
290        @return a new tko kernel object.
291        """
292
293        fields_dict = self.get_trivial_attr(kernel, self.kernel_type_dict)
294
295        return models.kernel(fields_dict['base'], [], fields_dict['kernel_hash'])
296
297
298    def set_pb_kernel(self, tko_kernel, pb_kernel):
299        """Set a specific kernel of a test.
300
301        Takes the same form of all the other setting methods.  It
302        seperates the string variables from the int variables and set
303        them safely.
304
305        @param
306        tko_kernel: a tko kernel.
307        pb_kernel: an empty protocol buffer kernel.
308
309        """
310
311        self.set_trivial_attr(tko_kernel, pb_kernel, self.kernel_type_dict)
312
313
314    def get_tko_iteration(self, iteration):
315        """Creates a new tko iteration with the data in the provided
316        pb iteration.
317
318        Uses the data in the pb iteration and the models.iteration
319        constructor to create a new tko iterations
320
321        @param
322        iteration: a pb iteration instance
323
324        @return a tko iteration instance with the same data.
325        """
326
327        fields_dict = self.get_trivial_attr(iteration,
328                                            self.iteration_type_dict)
329
330        fields_dict['attr_keyval'] = dict((keyval.name, keyval.value)
331                                          for keyval in iteration.attr_keyval)
332
333        fields_dict['perf_keyval'] = dict((keyval.name, keyval.value)
334                                          for keyval in iteration.perf_keyval)
335
336        return models.iteration(fields_dict['index'],
337                                fields_dict['attr_keyval'],
338                                fields_dict['perf_keyval'])
339
340
341    def set_pb_iteration(self, tko_iteration, pb_iteration):
342        """Sets all fields for a particular iteration.
343
344        Takes same form as all the other setting methods. Sets int,
345        str and datetime variables safely.
346
347        @param
348        tko_iteration: a tko test iteration.
349        pb_iteration: an empty pb test iteration.
350
351        """
352
353        self.set_trivial_attr(tko_iteration, pb_iteration,
354                              self.iteration_type_dict)
355
356        for key, val in tko_iteration.attr_keyval.iteritems():
357            newkeyval = pb_iteration.attr_keyval.add()
358            newkeyval.name = key
359            newkeyval.value = str(val)
360
361        for key, val in tko_iteration.perf_keyval.iteritems():
362            newkeyval = pb_iteration.perf_keyval.add()
363            newkeyval.name = key
364            newkeyval.value = str(val)
365
366
367    def get_trivial_attr(self, obj, objdict):
368        """Get all trivial attributes from the object.
369
370        This function is used to extract attributes from a pb job. The
371        dictionary specifies the types of each attribute in each tko
372        class.
373
374        @param
375        obj: the pb object that is being extracted.
376        objdict: the dict that specifies the type.
377
378        @return a dict of each attr name and it's corresponding value.
379        """
380
381        resultdict = {}
382        for field, field_type in objdict.items():
383            value = getattr(obj, field)
384            if field_type in (str, int, long):
385                resultdict[field] = field_type(value)
386            elif field_type == datetime:
387                resultdict[field] = (
388                            datetime.fromtimestamp(value/1000.0))
389
390        return resultdict
391
392
393    def set_trivial_attr(self, tko_obj, pb_obj, objdict):
394        """Sets all the easy attributes appropriately according to the
395        type.
396
397        This function is used to set all the trivial attributes
398        provided by objdict, the dictionary that specifies the types
399        of each attribute in each tko class.
400
401        @param
402        tko_obj: the original object that has the data being copied.
403        pb_obj: the new pb object that is being copied into.
404        objdict: specifies the type of each attribute in the class we
405        are working with.
406
407        """
408        for attr, attr_type in objdict.iteritems():
409            if attr_type == datetime:
410                t = getattr(tko_obj, attr)
411                if not t:
412                    self.set_attr_safely(pb_obj, attr, t, int)
413                else:
414                    t = mktime(t.timetuple()) + 1e-6 * t.microsecond
415                    setattr(pb_obj, attr, long(t*1000))
416            else:
417                value = getattr(tko_obj, attr)
418                self.set_attr_safely(pb_obj, attr, value, attr_type)
419
420
421    def set_attr_safely(self, var, attr, value, vartype):
422        """Sets a particular attribute of var if the provided value is
423        not None.
424
425        Checks if value is None. If not, set the attribute of the var
426        to be the default value. This is necessary for the special
427        required fields of the protocol buffer.
428
429        @param
430        var: the variable of which one of the attribute is being set.
431        attr: the attribute that is being set.
432        value: the value that is being checked
433        vartype: the expected type of the attr
434
435        """
436
437        supported_types = [int, long, str]
438        if vartype in supported_types:
439            if value is None:
440                value = vartype()
441            else:
442                assert isinstance(value, vartype), (
443                'Unexpected type %s for attr %s, should be %s' %
444                (type(value), attr, vartype))
445
446            setattr(var, attr, value)
447