results_cache.py revision f84bd3b7b64deba46192ae6d15a712ab66e90a6e 
1#!/usr/bin/python
2#
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5import getpass
6import glob
7import hashlib
8import os
9import pickle
10import re
11import tempfile
12
13from image_checksummer import ImageChecksummer
14from utils import command_executer
15from utils import logger
16from utils import misc
17
18
19SCRATCH_DIR = "/home/%s/cros_scratch" % getpass.getuser()
20RESULTS_FILE = "results.txt"
21AUTOTEST_TARBALL = "autotest.tbz2"
22PERF_RESULTS_FILE = "perf-results.txt"
23
24
25class Result(object):
26  """ This class manages what exactly is stored inside the cache without knowing
27  what the key of the cache is. For runs with perf, it stores perf.data,
28  perf.report, etc. The key generation is handled by the ResultsCache class.
29  """
30  def __init__(self, logger):
31    self._logger = logger
32    self._ce = command_executer.GetCommandExecuter(self._logger)
33    self._temp_dir = None
34
35  def _CopyFilesTo(self, dest_dir, files_to_copy):
36    file_index = 0
37    for file_to_copy in files_to_copy:
38      if not os.path.isdir(dest_dir):
39        command = "mkdir -p %s" % dest_dir
40        self._ce.RunCommand(command)
41      dest_file = os.path.join(dest_dir,
42                               ("%s.%s" % (os.path.basename(file_to_copy),
43                                           file_index)))
44      ret = self._ce.CopyFiles(file_to_copy,
45                               dest_file,
46                               recursive=False)
47      if ret:
48        raise Exception("Could not copy results file: %s" % file_to_copy)
49
50  def CopyResultsTo(self, dest_dir):
51    self._CopyFilesTo(dest_dir, self.perf_data_files)
52    self._CopyFilesTo(dest_dir, self.perf_report_files)
53
54  def _GetKeyvals(self):
55    command = "find %s -regex .*results/keyval$" % self.results_dir
56    [ret, out, err] = self._ce.RunCommand(command, return_output=True)
57    keyvals_dict = {}
58    for f in out.splitlines():
59      keyvals = open(f, "r").read()
60      keyvals_dict.update(self._ParseKeyvals(keyvals))
61
62    return keyvals_dict
63
64  def _ParseKeyvals(self, keyvals):
65    keyval_dict = {}
66    for l in keyvals.splitlines():
67      l = l.strip()
68      if l:
69        key, val = l.split("=")
70        keyval_dict[key] = val
71    return keyval_dict
72
73  def _GetResultsDir(self):
74    mo = re.search("Results placed in (\S+)", self.out)
75    if mo:
76      result = mo.group(1)
77      return result
78    raise Exception("Could not find results directory.")
79
80  def _FindFilesInResultsDir(self, find_args):
81    command = "find %s %s" % (self.results_dir,
82                              find_args)
83    ret, out, err = self._ce.RunCommand(command, return_output=True)
84    if ret:
85      raise Exception("Could not run find command!")
86    return out
87
88  def _GetPerfDataFiles(self):
89    return self._FindFilesInResultsDir("-name perf.data").splitlines()
90
91  def _GetPerfReportFiles(self):
92    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
93
94  def _GeneratePerfReportFiles(self):
95    perf_report_files = []
96    for perf_data_file in self.perf_data_files:
97      # Generate a perf.report and store it side-by-side with the perf.data
98      # file.
99      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
100                                                       perf_data_file)
101      perf_report_file = "%s.report" % perf_data_file
102      if os.path.exists(perf_report_file):
103        raise Exception("Perf report file already exists: %s" %
104                        perf_report_file)
105      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
106                                                   perf_report_file)
107      command = ("/usr/sbin/perf report "
108                 "--symfs /build/%s "
109                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
110                 "--kallsyms /build/%s/boot/System.map-* "
111                 "-i %s --stdio "
112                 "| head -n1000 "
113                 "| tee %s" %
114                 (self._board,
115                  self._board,
116                  self._board,
117                  chroot_perf_data_file,
118                  chroot_perf_report_file))
119      ret, out, err = self._ce.ChrootRunCommand(self._chromeos_root,
120                                                command,
121                                                return_output=True)
122      # Add a keyval to the dictionary for the events captured.
123      perf_report_files.append(
124          misc.GetOutsideChrootPath(self._chromeos_root,
125                                    chroot_perf_report_file))
126    return perf_report_files
127
128  def _GatherPerfResults(self):
129    report_id = 0
130    for perf_report_file in self.perf_report_files:
131      with open(perf_report_file, "r") as f:
132        report_contents = f.read()
133        for group in re.findall("Events: (\S+) (\S+)", report_contents):
134          num_events = group[0]
135          event_name = group[1]
136          key = "perf_%s_%s" % (report_id, event_name)
137          value = str(misc.UnitToNumber(num_events))
138          self.keyvals[key] = value
139
140  def _PopulateFromRun(self, chromeos_root, board, out, err, retval):
141    self._chromeos_root = chromeos_root
142    self._board = board
143    self.out = out
144    self.err = err
145    self.retval = retval
146    self.chroot_results_dir = self._GetResultsDir()
147    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
148                                                 self.chroot_results_dir)
149    self.perf_data_files = self._GetPerfDataFiles()
150    # Include all perf.report data in table.
151    self.perf_report_files = self._GeneratePerfReportFiles()
152    # TODO(asharif): Do something similar with perf stat.
153
154    # Grab keyvals from the directory.
155    self._ProcessResults()
156
157  def _ProcessResults(self):
158    # Note that this function doesn't know anything about whether there is a
159    # cache hit or miss. It should process results agnostic of the cache hit
160    # state.
161    self.keyvals = self._GetKeyvals()
162    self.keyvals["retval"] = self.retval
163    # Generate report from all perf.data files.
164    # Now parse all perf report files and include them in keyvals.
165    self._GatherPerfResults()
166
167  def _PopulateFromCacheDir(self, cache_dir):
168    # Read in everything from the cache directory.
169    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
170      self.out = pickle.load(f)
171      self.err = pickle.load(f)
172      self.retval = pickle.load(f)
173
174    # Untar the tarball to a temporary directory
175    self._temp_dir = tempfile.mkdtemp()
176    command = ("cd %s && tar xf %s" %
177               (self._temp_dir,
178                os.path.join(cache_dir, AUTOTEST_TARBALL)))
179    ret = self._ce.RunCommand(command)
180    if ret:
181      raise Exception("Could not untar cached tarball")
182    self.results_dir = self._temp_dir
183    self.perf_data_files = self._GetPerfDataFiles()
184    self.perf_report_files = self._GetPerfReportFiles()
185    self._ProcessResults()
186
187  def CleanUp(self):
188    if self._temp_dir:
189      command = "rm -rf %s" % self._temp_dir
190      self._ce.RunCommand(command)
191
192
193  def StoreToCacheDir(self, cache_dir):
194    # Create the dir if it doesn't exist.
195    command = "mkdir -p %s" % cache_dir
196    ret = self._ce.RunCommand(command)
197    if ret:
198      raise Exception("Could not create cache dir: %s" % cache_dir)
199    # Store to the cache directory.
200    with open(os.path.join(cache_dir, RESULTS_FILE), "w") as f:
201      pickle.dump(self.out, f)
202      pickle.dump(self.err, f)
203      pickle.dump(self.retval, f)
204
205    tarball = os.path.join(cache_dir, AUTOTEST_TARBALL)
206    command = ("cd %s && tar cjf %s ." % (self.results_dir, tarball))
207    ret = self._ce.RunCommand(command)
208    if ret:
209      raise Exception("Couldn't store autotest output directory.")
210
211  @classmethod
212  def CreateFromRun(cls, logger, chromeos_root, board, out, err, retval):
213    result = cls(logger)
214    result._PopulateFromRun(chromeos_root, board, out, err, retval)
215    return result
216
217  @classmethod
218  def CreateFromCacheHit(cls, logger, cache_dir):
219    result = cls(logger)
220    try:
221      result._PopulateFromCacheDir(cache_dir)
222    except Exception as e:
223      logger.LogError("Exception while using cache: %s" % e)
224      return None
225    return result
226
227
228class CacheConditions(object):
229  # Cache hit only if the result file exists.
230  CACHE_FILE_EXISTS = 0
231
232  # Cache hit if the ip address of the cached result and the new run match.
233  REMOTES_MATCH = 1
234
235  # Cache hit if the image checksum of the cached result and the new run match.
236  CHECKSUMS_MATCH = 2
237
238  # Cache hit only if the cached result was successful
239  RUN_SUCCEEDED = 3
240
241  # Never a cache hit.
242  FALSE = 4
243
244  # Cache hit if the image path matches the cached image path.
245  IMAGE_PATH_MATCH = 5
246
247
248class ResultsCache(object):
249  """ This class manages the key of the cached runs without worrying about what
250  is exactly stored (value). The value generation is handled by the Results
251  class.
252  """
253  CACHE_VERSION = 3
254  def Init(self, chromeos_image, chromeos_root, autotest_name, iteration,
255           autotest_args, remote, board, cache_conditions,
256           logger_to_use):
257    self.chromeos_image = chromeos_image
258    self.chromeos_root = chromeos_root
259    self.autotest_name = autotest_name
260    self.iteration = iteration
261    self.autotest_args = autotest_args,
262    self.remote = remote
263    self.board = board
264    self.cache_conditions = cache_conditions
265    self._logger = logger_to_use
266    self._ce = command_executer.GetCommandExecuter(self._logger)
267
268  def _GetCacheDirForRead(self):
269    glob_path = self._FormCacheDir(self._GetCacheKeyList(True))
270    matching_dirs = glob.glob(glob_path)
271
272    if matching_dirs:
273      # Cache file found.
274      if len(matching_dirs) > 1:
275        self._logger.LogError("Multiple compatible cache files: %s." %
276                              " ".join(matching_dirs))
277      return matching_dirs[0]
278    else:
279      return None
280
281  def _GetCacheDirForWrite(self):
282    return self._FormCacheDir(self._GetCacheKeyList(False))
283
284  def _FormCacheDir(self, list_of_strings):
285    cache_key = " ".join(list_of_strings)
286    cache_dir = misc.GetFilenameFromString(cache_key)
287    cache_path = os.path.join(SCRATCH_DIR, cache_dir)
288    return cache_path
289
290  def _GetCacheKeyList(self, read):
291    if read and CacheConditions.REMOTES_MATCH not in self.cache_conditions:
292      remote = "*"
293    else:
294      remote = self.remote
295    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
296      checksum = "*"
297    else:
298      checksum = ImageChecksummer().Checksum(self.chromeos_image)
299
300    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
301      image_path_checksum = "*"
302    else:
303      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
304
305    autotest_args_checksum = hashlib.md5(
306                             "".join(self.autotest_args)).hexdigest()
307
308    return (image_path_checksum,
309            self.autotest_name, str(self.iteration),
310            autotest_args_checksum,
311            checksum,
312            remote,
313            str(self.CACHE_VERSION))
314
315  def ReadResult(self):
316    if CacheConditions.FALSE in self.cache_conditions:
317      return None
318    cache_dir = self._GetCacheDirForRead()
319
320    if not cache_dir:
321      return None
322
323    if not os.path.isdir(cache_dir):
324      return None
325
326    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
327
328    result = Result.CreateFromCacheHit(self._logger, cache_dir)
329
330    if not result:
331      return None
332
333    if (result.retval == 0 or
334        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
335      return result
336
337    return None
338
339  def StoreResult(self, result):
340    cache_dir = self._GetCacheDirForWrite()
341    result.StoreToCacheDir(cache_dir)
342
343
344class MockResultsCache(object):
345  def Init(self, *args):
346    pass
347
348  def ReadResult(self):
349    return Result("Results placed in /tmp/test", "", 0)
350
351  def StoreResult(self, result):
352    pass
353