results_cache.py revision 006a5b0cd4ed149e7774ef2d3a29a73d0cc3cee8 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import getpass
8import glob
9import hashlib
10import os
11import pickle
12import re
13import tempfile
14
15from utils import command_executer
16from utils import misc
17
18from image_checksummer import ImageChecksummer
19
20SCRATCH_BASE = "/home/%s/cros_scratch"
21SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
22RESULTS_FILE = "results.txt"
23MACHINE_FILE = "machine.txt"
24AUTOTEST_TARBALL = "autotest.tbz2"
25PERF_RESULTS_FILE = "perf-results.txt"
26
27
28class Result(object):
29  """ This class manages what exactly is stored inside the cache without knowing
30  what the key of the cache is. For runs with perf, it stores perf.data,
31  perf.report, etc. The key generation is handled by the ResultsCache class.
32  """
33
34  def __init__(self, chromeos_root, logger, label_name):
35    self._chromeos_root = chromeos_root
36    self._logger = logger
37    self._ce = command_executer.GetCommandExecuter(self._logger)
38    self._temp_dir = None
39    self.label_name = label_name
40
41  def _CopyFilesTo(self, dest_dir, files_to_copy):
42    file_index = 0
43    for file_to_copy in files_to_copy:
44      if not os.path.isdir(dest_dir):
45        command = "mkdir -p %s" % dest_dir
46        self._ce.RunCommand(command)
47      dest_file = os.path.join(dest_dir,
48                               ("%s.%s" % (os.path.basename(file_to_copy),
49                                           file_index)))
50      ret = self._ce.CopyFiles(file_to_copy,
51                               dest_file,
52                               recursive=False)
53      if ret:
54        raise Exception("Could not copy results file: %s" % file_to_copy)
55
56  def CopyResultsTo(self, dest_dir):
57    self._CopyFilesTo(dest_dir, self.perf_data_files)
58    self._CopyFilesTo(dest_dir, self.perf_report_files)
59
60  def _GetKeyvals(self):
61    results_in_chroot = os.path.join(self._chromeos_root,
62                                     "chroot", "tmp")
63    if not self._temp_dir:
64      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
65      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
66      self._ce.RunCommand(command)
67
68    command = ("python generate_test_report --no-color --csv %s" %
69               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
70    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
71                                            command,
72                                            return_output=True)
73    keyvals_dict = {}
74    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
75                                                 self._temp_dir)
76    for line in out.splitlines():
77      tokens = re.split("=|,", line)
78      key = tokens[-2]
79      if key.startswith(tmp_dir_in_chroot):
80        key = key[len(tmp_dir_in_chroot) + 1:]
81      value = tokens[-1]
82      keyvals_dict[key] = value
83
84    return keyvals_dict
85
86  def _GetResultsDir(self):
87    mo = re.search(r"Results placed in (\S+)", self.out)
88    if mo:
89      result = mo.group(1)
90      return result
91    raise Exception("Could not find results directory.")
92
93  def _FindFilesInResultsDir(self, find_args):
94    command = "find %s %s" % (self.results_dir,
95                              find_args)
96    ret, out, _ = self._ce.RunCommand(command, return_output=True)
97    if ret:
98      raise Exception("Could not run find command!")
99    return out
100
101  def _GetPerfDataFiles(self):
102    return self._FindFilesInResultsDir("-name perf.data").splitlines()
103
104  def _GetPerfReportFiles(self):
105    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
106
107  def _GeneratePerfReportFiles(self):
108    perf_report_files = []
109    for perf_data_file in self.perf_data_files:
110      # Generate a perf.report and store it side-by-side with the perf.data
111      # file.
112      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
113                                                       perf_data_file)
114      perf_report_file = "%s.report" % perf_data_file
115      if os.path.exists(perf_report_file):
116        raise Exception("Perf report file already exists: %s" %
117                        perf_report_file)
118      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
119                                                         perf_report_file)
120      command = ("/usr/sbin/perf report "
121                 "-n "
122                 "--symfs /build/%s "
123                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
124                 "--kallsyms /build/%s/boot/System.map-* "
125                 "-i %s --stdio "
126                 "> %s" %
127                 (self._board,
128                  self._board,
129                  self._board,
130                  chroot_perf_data_file,
131                  chroot_perf_report_file))
132      self._ce.ChrootRunCommand(self._chromeos_root,
133                                command)
134
135      # Add a keyval to the dictionary for the events captured.
136      perf_report_files.append(
137          misc.GetOutsideChrootPath(self._chromeos_root,
138                                    chroot_perf_report_file))
139    return perf_report_files
140
141  def _GatherPerfResults(self):
142    report_id = 0
143    for perf_report_file in self.perf_report_files:
144      with open(perf_report_file, "r") as f:
145        report_contents = f.read()
146        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
147          num_events = group[0]
148          event_name = group[1]
149          key = "perf_%s_%s" % (report_id, event_name)
150          value = str(misc.UnitToNumber(num_events))
151          self.keyvals[key] = value
152
153  def _PopulateFromRun(self, board, out, err, retval):
154    self._board = board
155    self.out = out
156    self.err = err
157    self.retval = retval
158    self.chroot_results_dir = self._GetResultsDir()
159    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
160                                                 self.chroot_results_dir)
161    self.perf_data_files = self._GetPerfDataFiles()
162    # Include all perf.report data in table.
163    self.perf_report_files = self._GeneratePerfReportFiles()
164    # TODO(asharif): Do something similar with perf stat.
165
166    # Grab keyvals from the directory.
167    self._ProcessResults()
168
169  def _ProcessResults(self):
170    # Note that this function doesn't know anything about whether there is a
171    # cache hit or miss. It should process results agnostic of the cache hit
172    # state.
173    self.keyvals = self._GetKeyvals()
174    self.keyvals["retval"] = self.retval
175    # Generate report from all perf.data files.
176    # Now parse all perf report files and include them in keyvals.
177    self._GatherPerfResults()
178
179  def _PopulateFromCacheDir(self, cache_dir):
180    # Read in everything from the cache directory.
181    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
182      self.out = pickle.load(f)
183      self.err = pickle.load(f)
184      self.retval = pickle.load(f)
185
186    # Untar the tarball to a temporary directory
187    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
188                                                       "chroot", "tmp"))
189
190    command = ("cd %s && tar xf %s" %
191               (self._temp_dir,
192                os.path.join(cache_dir, AUTOTEST_TARBALL)))
193    ret = self._ce.RunCommand(command)
194    if ret:
195      raise Exception("Could not untar cached tarball")
196    self.results_dir = self._temp_dir
197    self.perf_data_files = self._GetPerfDataFiles()
198    self.perf_report_files = self._GetPerfReportFiles()
199    self._ProcessResults()
200
201  def CleanUp(self, rm_chroot_tmp):
202    if rm_chroot_tmp:
203      command = "rm -rf %s" % self.results_dir
204      self._ce.RunCommand(command)
205    if self._temp_dir:
206      command = "rm -rf %s" % self._temp_dir
207      self._ce.RunCommand(command)
208
209  def StoreToCacheDir(self, cache_dir, machine_manager):
210    # Create the dir if it doesn't exist.
211    temp_dir = tempfile.mkdtemp()
212
213    # Store to the temp directory.
214    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
215      pickle.dump(self.out, f)
216      pickle.dump(self.err, f)
217      pickle.dump(self.retval, f)
218
219    tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
220    command = ("cd %s && "
221               "tar "
222               "--exclude=var/spool "
223               "--exclude=var/log "
224               "-cjf %s ." % (self.results_dir, tarball))
225    ret = self._ce.RunCommand(command)
226    if ret:
227      raise Exception("Couldn't store autotest output directory.")
228    # Store machine info.
229    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
230    # this function.
231    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
232      f.write(machine_manager.machine_checksum_string[self.label_name])
233
234    if os.path.exists(cache_dir):
235      command = "rm -rf {0}".format(cache_dir)
236      self._ce.RunCommand(command)
237
238    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
239    command += "chmod g+x {0} && ".format(temp_dir)
240    command += "mv {0} {1}".format(temp_dir, cache_dir)
241    ret = self._ce.RunCommand(command)
242    if ret:
243      command = "rm -rf {0}".format(temp_dir)
244      self._ce.RunCommand(command)
245      raise Exception("Could not move dir %s to dir %s" %
246                      (temp_dir, cache_dir))
247
248  @classmethod
249  def CreateFromRun(cls, logger, chromeos_root, board, label_name,
250                    out, err, retval):
251    result = cls(chromeos_root, logger, label_name)
252    result._PopulateFromRun(board, out, err, retval)
253    return result
254
255  @classmethod
256  def CreateFromCacheHit(cls, chromeos_root, logger, cache_dir, label_name):
257    result = cls(chromeos_root, logger, label_name)
258    try:
259      result._PopulateFromCacheDir(cache_dir)
260    except Exception as e:
261      logger.LogError("Exception while using cache: %s" % e)
262      return None
263    return result
264
265
266class CacheConditions(object):
267  # Cache hit only if the result file exists.
268  CACHE_FILE_EXISTS = 0
269
270  # Cache hit if the checksum of cpuinfo and totalmem of
271  # the cached result and the new run match.
272  MACHINES_MATCH = 1
273
274  # Cache hit if the image checksum of the cached result and the new run match.
275  CHECKSUMS_MATCH = 2
276
277  # Cache hit only if the cached result was successful
278  RUN_SUCCEEDED = 3
279
280  # Never a cache hit.
281  FALSE = 4
282
283  # Cache hit if the image path matches the cached image path.
284  IMAGE_PATH_MATCH = 5
285
286  # Cache hit if the uuid of hard disk mataches the cached one
287
288  SAME_MACHINE_MATCH = 6
289
290
291class ResultsCache(object):
292  """ This class manages the key of the cached runs without worrying about what
293  is exactly stored (value). The value generation is handled by the Results
294  class.
295  """
296  CACHE_VERSION = 6
297
298  def Init(self, chromeos_image, chromeos_root, autotest_name, iteration,
299           autotest_args, machine_manager, board, cache_conditions,
300           logger_to_use, label, share_users):
301    self.chromeos_image = chromeos_image
302    self.chromeos_root = chromeos_root
303    self.autotest_name = autotest_name
304    self.iteration = iteration
305    self.autotest_args = autotest_args,
306    self.board = board
307    self.cache_conditions = cache_conditions
308    self.machine_manager = machine_manager
309    self._logger = logger_to_use
310    self._ce = command_executer.GetCommandExecuter(self._logger)
311    self.label = label
312    self.share_users = share_users
313
314  def _GetCacheDirForRead(self):
315    matching_dirs = []
316    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
317      matching_dirs += glob.glob(glob_path)
318
319    if matching_dirs:
320      # Cache file found.
321      return matching_dirs[0]
322    else:
323      return None
324
325  def _GetCacheDirForWrite(self):
326    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
327
328  def _FormCacheDir(self, list_of_strings):
329    cache_key = " ".join(list_of_strings)
330    cache_dir = misc.GetFilenameFromString(cache_key)
331    if self.label.cache_dir:
332      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
333      cache_path = [os.path.join(cache_home, cache_dir)]
334    else:
335      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
336
337    for i in [x.strip() for x in self.share_users.split(",")]:
338      path = SCRATCH_BASE % i
339      cache_path.append(os.path.join(path, cache_dir))
340
341    return cache_path
342
343  def _GetCacheKeyList(self, read):
344    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
345      machine_checksum = "*"
346    else:
347      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
348    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
349      checksum = "*"
350    else:
351      checksum = ImageChecksummer().Checksum(self.label)
352
353    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
354      image_path_checksum = "*"
355    else:
356      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
357
358    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
359      machine_id_checksum = "*"
360    else:
361      for machine in self.machine_manager.GetMachines(self.label):
362        if machine.name == self.label.remote[0]:
363          machine_id_checksum = machine.machine_id_checksum
364          break
365
366    autotest_args_checksum = hashlib.md5(
367        "".join(self.autotest_args)).hexdigest()
368    return (image_path_checksum,
369            self.autotest_name, str(self.iteration),
370            autotest_args_checksum,
371            checksum,
372            machine_checksum,
373            machine_id_checksum,
374            str(self.CACHE_VERSION))
375
376  def ReadResult(self):
377    if CacheConditions.FALSE in self.cache_conditions:
378      return None
379    cache_dir = self._GetCacheDirForRead()
380
381    if not cache_dir:
382      return None
383
384    if not os.path.isdir(cache_dir):
385      return None
386
387    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
388
389    result = Result.CreateFromCacheHit(self.chromeos_root,
390                                       self._logger, cache_dir, self.label.name)
391
392    if not result:
393      return None
394
395    if (result.retval == 0 or
396        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
397      return result
398
399    return None
400
401  def StoreResult(self, result):
402    cache_dir = self._GetCacheDirForWrite()
403    result.StoreToCacheDir(cache_dir, self.machine_manager)
404
405
406class MockResultsCache(ResultsCache):
407  def Init(self, *args):
408    pass
409
410  def ReadResult(self):
411    return None
412
413  def StoreResult(self, result):
414    pass
415
416
417class MockResult(Result):
418  def _PopulateFromRun(self, out, err, retval):
419    self.out = out
420    self.err = err
421    self.retval = retval
422
423