results_cache.py revision 5f3ccbddaaf5f4a64431c9e68faf83a348ecdfdd 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to deal with result cache."""
8
9import getpass
10import glob
11import hashlib
12import os
13import pickle
14import re
15import tempfile
16import json
17
18from utils import command_executer
19from utils import misc
20
21from image_checksummer import ImageChecksummer
22
23SCRATCH_BASE = "/home/%s/cros_scratch"
24SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
25RESULTS_FILE = "results.txt"
26MACHINE_FILE = "machine.txt"
27AUTOTEST_TARBALL = "autotest.tbz2"
28PERF_RESULTS_FILE = "perf-results.txt"
29
30
31class Result(object):
32  """ This class manages what exactly is stored inside the cache without knowing
33  what the key of the cache is. For runs with perf, it stores perf.data,
34  perf.report, etc. The key generation is handled by the ResultsCache class.
35  """
36
37  def __init__(self, logger, label):
38    self._chromeos_root = label.chromeos_root
39    self._logger = logger
40    self._ce = command_executer.GetCommandExecuter(self._logger)
41    self._temp_dir = None
42    self.label = label
43    self.results_dir = None
44    self.perf_data_files = []
45    self.perf_report_files = []
46
47  def _CopyFilesTo(self, dest_dir, files_to_copy):
48    file_index = 0
49    for file_to_copy in files_to_copy:
50      if not os.path.isdir(dest_dir):
51        command = "mkdir -p %s" % dest_dir
52        self._ce.RunCommand(command)
53      dest_file = os.path.join(dest_dir,
54                               ("%s.%s" % (os.path.basename(file_to_copy),
55                                           file_index)))
56      ret = self._ce.CopyFiles(file_to_copy,
57                               dest_file,
58                               recursive=False)
59      if ret:
60        raise Exception("Could not copy results file: %s" % file_to_copy)
61
62  def CopyResultsTo(self, dest_dir):
63    self._CopyFilesTo(dest_dir, self.perf_data_files)
64    self._CopyFilesTo(dest_dir, self.perf_report_files)
65
66  def _GetNewKeyvals(self, keyvals_dict):
67    results_files = self._GetDataMeasurementsFiles()
68    for f in results_files:
69      data_filename = os.path.join(self._chromeos_root, "/tmp",
70                                  self._temp_dir, f)
71      if os.path.exists(data_filename):
72        with open(data_filename, "r") as data_file:
73          lines = data_file.readlines()
74          for line in lines:
75            tmp_dict = json.loads(line)
76            key = tmp_dict['description']
77            val = tmp_dict['value']
78            keyvals_dict[key] = val
79
80    return keyvals_dict
81
82
83  def _GetKeyvals(self):
84    results_in_chroot = os.path.join(self._chromeos_root,
85                                     "chroot", "tmp")
86    if not self._temp_dir:
87      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
88      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
89      self._ce.RunCommand(command)
90
91    command = ("python generate_test_report --no-color --csv %s" %
92               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
93    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
94                                            command,
95                                            return_output=True)
96    keyvals_dict = {}
97    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
98                                                 self._temp_dir)
99    for line in out.splitlines():
100      tokens = re.split("=|,", line)
101      key = tokens[-2]
102      if key.startswith(tmp_dir_in_chroot):
103        key = key[len(tmp_dir_in_chroot) + 1:]
104      value = tokens[-1]
105      keyvals_dict[key] = value
106
107    # Check to see if there is a perf_measurements file and get the
108    # data from it if so.
109    return self._GetNewKeyvals(keyvals_dict)
110
111  def _GetResultsDir(self):
112    mo = re.search(r"Results placed in (\S+)", self.out)
113    if mo:
114      result = mo.group(1)
115      return result
116    raise Exception("Could not find results directory.")
117
118  def _FindFilesInResultsDir(self, find_args):
119    if not self.results_dir:
120      return None
121
122    command = "find %s %s" % (self.results_dir,
123                              find_args)
124    ret, out, _ = self._ce.RunCommand(command, return_output=True)
125    if ret:
126      raise Exception("Could not run find command!")
127    return out
128
129  def _GetPerfDataFiles(self):
130    return self._FindFilesInResultsDir("-name perf.data").splitlines()
131
132  def _GetPerfReportFiles(self):
133    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
134
135  def _GetDataMeasurementsFiles(self):
136    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()
137
138  def _GeneratePerfReportFiles(self):
139    perf_report_files = []
140    for perf_data_file in self.perf_data_files:
141      # Generate a perf.report and store it side-by-side with the perf.data
142      # file.
143      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
144                                                       perf_data_file)
145      perf_report_file = "%s.report" % perf_data_file
146      if os.path.exists(perf_report_file):
147        raise Exception("Perf report file already exists: %s" %
148                        perf_report_file)
149      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
150                                                         perf_report_file)
151      command = ("/usr/sbin/perf report "
152                 "-n "
153                 "--symfs /build/%s "
154                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
155                 "--kallsyms /build/%s/boot/System.map-* "
156                 "-i %s --stdio "
157                 "> %s" %
158                 (self._board,
159                  self._board,
160                  self._board,
161                  chroot_perf_data_file,
162                  chroot_perf_report_file))
163      self._ce.ChrootRunCommand(self._chromeos_root,
164                                command)
165
166      # Add a keyval to the dictionary for the events captured.
167      perf_report_files.append(
168          misc.GetOutsideChrootPath(self._chromeos_root,
169                                    chroot_perf_report_file))
170    return perf_report_files
171
172  def _GatherPerfResults(self):
173    report_id = 0
174    for perf_report_file in self.perf_report_files:
175      with open(perf_report_file, "r") as f:
176        report_contents = f.read()
177        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
178          num_events = group[0]
179          event_name = group[1]
180          key = "perf_%s_%s" % (report_id, event_name)
181          value = str(misc.UnitToNumber(num_events))
182          self.keyvals[key] = value
183
184  def _PopulateFromRun(self, out, err, retval):
185    self._board = self.label.board
186    self.out = out
187    self.err = err
188    self.retval = retval
189    self.chroot_results_dir = self._GetResultsDir()
190    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
191                                                 self.chroot_results_dir)
192    self.perf_data_files = self._GetPerfDataFiles()
193    # Include all perf.report data in table.
194    self.perf_report_files = self._GeneratePerfReportFiles()
195    # TODO(asharif): Do something similar with perf stat.
196
197    # Grab keyvals from the directory.
198    self._ProcessResults()
199
200  def _ProcessResults(self):
201    # Note that this function doesn't know anything about whether there is a
202    # cache hit or miss. It should process results agnostic of the cache hit
203    # state.
204    self.keyvals = self._GetKeyvals()
205    self.keyvals["retval"] = self.retval
206    # Generate report from all perf.data files.
207    # Now parse all perf report files and include them in keyvals.
208    self._GatherPerfResults()
209
210  def _PopulateFromCacheDir(self, cache_dir):
211    # Read in everything from the cache directory.
212    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
213      self.out = pickle.load(f)
214      self.err = pickle.load(f)
215      self.retval = pickle.load(f)
216
217    # Untar the tarball to a temporary directory
218    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
219                                                       "chroot", "tmp"))
220
221    command = ("cd %s && tar xf %s" %
222               (self._temp_dir,
223                os.path.join(cache_dir, AUTOTEST_TARBALL)))
224    ret = self._ce.RunCommand(command)
225    if ret:
226      raise Exception("Could not untar cached tarball")
227    self.results_dir = self._temp_dir
228    self.perf_data_files = self._GetPerfDataFiles()
229    self.perf_report_files = self._GetPerfReportFiles()
230    self._ProcessResults()
231
232  def CleanUp(self, rm_chroot_tmp):
233    if rm_chroot_tmp and self.results_dir:
234      command = "rm -rf %s" % self.results_dir
235      self._ce.RunCommand(command)
236    if self._temp_dir:
237      command = "rm -rf %s" % self._temp_dir
238      self._ce.RunCommand(command)
239
240  def StoreToCacheDir(self, cache_dir, machine_manager):
241    # Create the dir if it doesn't exist.
242    temp_dir = tempfile.mkdtemp()
243
244    # Store to the temp directory.
245    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
246      pickle.dump(self.out, f)
247      pickle.dump(self.err, f)
248      pickle.dump(self.retval, f)
249
250    if self.results_dir:
251      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
252      command = ("cd %s && "
253                 "tar "
254                 "--exclude=var/spool "
255                 "--exclude=var/log "
256                 "-cjf %s ." % (self.results_dir, tarball))
257      ret = self._ce.RunCommand(command)
258      if ret:
259        raise Exception("Couldn't store autotest output directory.")
260    # Store machine info.
261    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
262    # this function.
263    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
264      f.write(machine_manager.machine_checksum_string[self.label.name])
265
266    if os.path.exists(cache_dir):
267      command = "rm -rf {0}".format(cache_dir)
268      self._ce.RunCommand(command)
269
270    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
271    command += "chmod g+x {0} && ".format(temp_dir)
272    command += "mv {0} {1}".format(temp_dir, cache_dir)
273    ret = self._ce.RunCommand(command)
274    if ret:
275      command = "rm -rf {0}".format(temp_dir)
276      self._ce.RunCommand(command)
277      raise Exception("Could not move dir %s to dir %s" %
278                      (temp_dir, cache_dir))
279
280  @classmethod
281  def CreateFromRun(cls, logger, label, out, err, retval, suite="pyauto"):
282    if suite == "telemetry":
283      result = TelemetryResult(logger, label)
284    else:
285      result = cls(logger, label)
286    result._PopulateFromRun(out, err, retval)
287    return result
288
289  @classmethod
290  def CreateFromCacheHit(cls, logger, label, cache_dir,
291                         suite="pyauto"):
292    if suite == "telemetry":
293      result = TelemetryResult(logger, label)
294    else:
295      result = cls(logger, label)
296    try:
297      result._PopulateFromCacheDir(cache_dir)
298
299    except Exception as e:
300      logger.LogError("Exception while using cache: %s" % e)
301      return None
302    return result
303
304
305class TelemetryResult(Result):
306
307  def __init__(self, logger, label):
308    super(TelemetryResult, self).__init__(logger, label)
309
310  def _PopulateFromRun(self, out, err, retval):
311    self.out = out
312    self.err = err
313    self.retval = retval
314
315    self._ProcessResults()
316
317  def _ProcessResults(self):
318    # The output is:
319    # url,average_commit_time (ms),...
320    # www.google.com,33.4,21.2,...
321    # We need to convert to this format:
322    # {"www.google.com:average_commit_time (ms)": "33.4",
323    #  "www.google.com:...": "21.2"}
324    # Added note:  Occasionally the output comes back
325    # with "JSON.stringify(window.automation.GetResults())" on
326    # the first line, and then the rest of the output as
327    # described above.
328
329    lines = self.out.splitlines()
330    self.keyvals = {}
331
332    if lines:
333      if lines[0].startswith("JSON.stringify"):
334        lines = lines[1:]
335
336    if not lines:
337      return
338    labels = lines[0].split(",")
339    for line in lines[1:]:
340      fields = line.split(",")
341      if len(fields) != len(labels):
342        continue
343      for i in range(1, len(labels)):
344        key = "%s %s" % (fields[0], labels[i])
345        value = fields[i]
346        self.keyvals[key] = value
347    self.keyvals["retval"] = self.retval
348
349  def _PopulateFromCacheDir(self, cache_dir):
350    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
351      self.out = pickle.load(f)
352      self.err = pickle.load(f)
353      self.retval = pickle.load(f)
354    self._ProcessResults()
355
356
357class CacheConditions(object):
358  # Cache hit only if the result file exists.
359  CACHE_FILE_EXISTS = 0
360
361  # Cache hit if the checksum of cpuinfo and totalmem of
362  # the cached result and the new run match.
363  MACHINES_MATCH = 1
364
365  # Cache hit if the image checksum of the cached result and the new run match.
366  CHECKSUMS_MATCH = 2
367
368  # Cache hit only if the cached result was successful
369  RUN_SUCCEEDED = 3
370
371  # Never a cache hit.
372  FALSE = 4
373
374  # Cache hit if the image path matches the cached image path.
375  IMAGE_PATH_MATCH = 5
376
377  # Cache hit if the uuid of hard disk mataches the cached one
378
379  SAME_MACHINE_MATCH = 6
380
381
382class ResultsCache(object):
383
384  """ This class manages the key of the cached runs without worrying about what
385  is exactly stored (value). The value generation is handled by the Results
386  class.
387  """
388  CACHE_VERSION = 6
389
390  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
391           test_args, machine_manager, board, cache_conditions,
392           logger_to_use, label, share_users, suite):
393    self.chromeos_image = chromeos_image
394    self.chromeos_root = chromeos_root
395    self.test_name = test_name
396    self.iteration = iteration
397    self.test_args = test_args,
398    self.board = board
399    self.cache_conditions = cache_conditions
400    self.machine_manager = machine_manager
401    self._logger = logger_to_use
402    self._ce = command_executer.GetCommandExecuter(self._logger)
403    self.label = label
404    self.share_users = share_users
405    self.suite = suite
406
407  def _GetCacheDirForRead(self):
408    matching_dirs = []
409    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
410      matching_dirs += glob.glob(glob_path)
411
412    if matching_dirs:
413      # Cache file found.
414      return matching_dirs[0]
415    else:
416      return None
417
418  def _GetCacheDirForWrite(self):
419    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
420
421  def _FormCacheDir(self, list_of_strings):
422    cache_key = " ".join(list_of_strings)
423    cache_dir = misc.GetFilenameFromString(cache_key)
424    if self.label.cache_dir:
425      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
426      cache_path = [os.path.join(cache_home, cache_dir)]
427    else:
428      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
429
430    for i in [x.strip() for x in self.share_users.split(",")]:
431      path = SCRATCH_BASE % i
432      cache_path.append(os.path.join(path, cache_dir))
433
434    return cache_path
435
436  def _GetCacheKeyList(self, read):
437    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
438      machine_checksum = "*"
439    else:
440      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
441    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
442      checksum = "*"
443    else:
444      checksum = ImageChecksummer().Checksum(self.label)
445
446    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
447      image_path_checksum = "*"
448    else:
449      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
450
451    machine_id_checksum = ""
452    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
453      machine_id_checksum = "*"
454    else:
455      for machine in self.machine_manager.GetMachines(self.label):
456        if machine.name == self.label.remote[0]:
457          machine_id_checksum = machine.machine_id_checksum
458          break
459
460    test_args_checksum = hashlib.md5(
461        "".join(self.test_args)).hexdigest()
462    return (image_path_checksum,
463            self.test_name, str(self.iteration),
464            test_args_checksum,
465            checksum,
466            machine_checksum,
467            machine_id_checksum,
468            str(self.CACHE_VERSION))
469
470  def ReadResult(self):
471    if CacheConditions.FALSE in self.cache_conditions:
472      cache_dir = self._GetCacheDirForWrite()
473      command = "rm -rf {0}".format(cache_dir)
474      self._ce.RunCommand(command)
475      return None
476    cache_dir = self._GetCacheDirForRead()
477
478    if not cache_dir:
479      return None
480
481    if not os.path.isdir(cache_dir):
482      return None
483
484    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
485    result = Result.CreateFromCacheHit(self._logger,
486                                       self.label,
487                                       cache_dir,
488                                       self.suite)
489    if not result:
490      return None
491
492    if (result.retval == 0 or
493        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
494      return result
495
496    return None
497
498  def StoreResult(self, result):
499    cache_dir = self._GetCacheDirForWrite()
500    result.StoreToCacheDir(cache_dir, self.machine_manager)
501
502
503class MockResultsCache(ResultsCache):
504  def Init(self, *args):
505    pass
506
507  def ReadResult(self):
508    return None
509
510  def StoreResult(self, result):
511    pass
512
513
514class MockResult(Result):
515  def _PopulateFromRun(self, out, err, retval):
516    self.out = out
517    self.err = err
518    self.retval = retval
519