results_cache.py revision ee988c87971c07b7e6767adb63d151ea37e363f2 
1#!/usr/bin/python
2#
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5import getpass
6import glob
7import hashlib
8import os
9import pickle
10import re
11
12from image_checksummer import ImageChecksummer
13from perf_processor import PerfProcessor
14from utils import command_executer
15from utils import logger
16from utils import misc
17
18
19SCRATCH_DIR = "/home/%s/cros_scratch" % getpass.getuser()
20RESULTS_FILE = "results.txt"
21AUTOTEST_TARBALL = "autotest.tbz2"
22PERF_RESULTS_FILE = "perf-results.txt"
23
24
25class Result(object):
26  def __init__(self, out, err, retval, keyvals):
27    self.out = out
28    self.err = err
29    self.retval = retval
30    self.keyvals = keyvals
31
32
33class CacheConditions(object):
34  # Cache hit only if the result file exists.
35  CACHE_FILE_EXISTS = 0
36
37  # Cache hit if the ip address of the cached result and the new run match.
38  REMOTES_MATCH = 1
39
40  # Cache hit if the image checksum of the cached result and the new run match.
41  CHECKSUMS_MATCH = 2
42
43  # Cache hit only if the cached result was successful
44  RUN_SUCCEEDED = 3
45
46  # Never a cache hit.
47  FALSE = 4
48
49  # Cache hit if the image path matches the cached image path.
50  IMAGE_PATH_MATCH = 5
51
52
53class ResultsCache(object):
54  CACHE_VERSION = 2
55  def Init(self, chromeos_image, chromeos_root, autotest_name, iteration,
56           autotest_args, remote, board, cache_conditions,
57           logger_to_use):
58    self.chromeos_image = chromeos_image
59    self.chromeos_root = chromeos_root
60    self.autotest_name = autotest_name
61    self.iteration = iteration
62    self.autotest_args = autotest_args,
63    self.remote = remote
64    self.board = board
65    self.cache_conditions = cache_conditions
66    self._logger = logger_to_use
67    self._ce = command_executer.GetCommandExecuter(self._logger)
68
69  def _GetCacheDirForRead(self):
70    glob_path = self._FormCacheDir(self._GetCacheKeyList(True))
71    matching_dirs = glob.glob(glob_path)
72
73    if matching_dirs:
74      # Cache file found.
75      if len(matching_dirs) > 1:
76        self._logger.LogError("Multiple compatible cache files: %s." %
77                              " ".join(matching_dirs))
78      return matching_dirs[0]
79    else:
80      return None
81
82  def _GetCacheDirForWrite(self):
83    return self._FormCacheDir(self._GetCacheKeyList(False))
84
85  def _FormCacheDir(self, list_of_strings):
86    cache_key = " ".join(list_of_strings)
87    cache_dir = misc.GetFilenameFromString(cache_key)
88    cache_path = os.path.join(SCRATCH_DIR, cache_dir)
89    return cache_path
90
91  def _GetCacheKeyList(self, read):
92    if read and CacheConditions.REMOTES_MATCH not in self.cache_conditions:
93      remote = "*"
94    else:
95      remote = self.remote
96    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
97      checksum = "*"
98    else:
99      checksum = ImageChecksummer().Checksum(self.chromeos_image)
100
101    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
102      image_path_checksum = "*"
103    else:
104      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
105
106    autotest_args_checksum = hashlib.md5(
107                             "".join(self.autotest_args)).hexdigest()
108
109    return (image_path_checksum,
110            self.autotest_name, str(self.iteration),
111            autotest_args_checksum,
112            checksum,
113            remote,
114            str(self.CACHE_VERSION))
115
116  def ReadResult(self):
117    if CacheConditions.FALSE in self.cache_conditions:
118      return None
119    cache_dir = self._GetCacheDirForRead()
120
121    if not cache_dir:
122      return None
123
124    try:
125      cache_file = os.path.join(cache_dir, RESULTS_FILE)
126
127      self._logger.LogOutput("Trying to read from cache file: %s" % cache_file)
128
129      with open(cache_file, "rb") as f:
130        result = pickle.load(f)
131
132        if (result.retval == 0 or
133            CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
134          return result
135
136    except Exception, e:
137      if CacheConditions.CACHE_FILE_EXISTS not in self.cache_conditions:
138        # Cache file not found but just return a failure.
139        return Result("", "", 1, {})
140      raise e
141
142  def StoreResult(self, result):
143    cache_dir = self._GetCacheDirForWrite()
144    cache_file = os.path.join(cache_dir, RESULTS_FILE)
145    command = "mkdir -p %s" % cache_dir
146    ret = self._ce.RunCommand(command)
147    assert ret == 0, "Couldn't create cache dir"
148    with open(cache_file, "wb") as f:
149      pickle.dump(result, f)
150
151  def StoreAutotestOutput(self, results_dir):
152    host_results_dir = os.path.join(self.chromeos_root, "chroot",
153                                    results_dir[1:])
154    tarball = os.path.join(self._GetCacheDirForWrite(), AUTOTEST_TARBALL)
155    command = ("cd %s && tar cjf %s ." % (host_results_dir, tarball))
156    ret = self._ce.RunCommand(command)
157    if ret:
158      raise Exception("Couldn't store autotest output directory.")
159
160  def ReadAutotestOutput(self, destination):
161    cache_dir = self._GetCacheDirForRead()
162    tarball = os.path.join(cache_dir, AUTOTEST_TARBALL)
163    if not os.path.exists(tarball):
164      raise Exception("Cached autotest tarball does not exist at '%s'." %
165                      tarball)
166    command = ("cd %s && tar xjf %s ." % (destination, tarball))
167    ret = self._ce.RunCommand(command)
168    if ret:
169      raise Exception("Couldn't read autotest output directory.")
170
171  def StorePerfResults(self, perf):
172    perf_path = os.path.join(self._GetCacheDirForWrite(), PERF_RESULTS_FILE)
173    with open(perf_path, "wb") as f:
174      pickle.dump(perf.report, f)
175      pickle.dump(perf.output, f)
176
177  def ReadPerfResults(self):
178    cache_dir = self._GetCacheDirForRead()
179    perf_path = os.path.join(cache_dir, PERF_RESULTS_FILE)
180    with open(perf_path, "rb") as f:
181      report = pickle.load(f)
182      output = pickle.load(f)
183
184    return PerfProcessor.PerfResults(report, output)
185
186
187class MockResultsCache(object):
188  def Init(self, *args):
189    pass
190
191  def ReadResult(self):
192    return Result("Results placed in /tmp/test", "", 0)
193
194  def StoreResult(self, result):
195    pass
196
197  def StoreAutotestOutput(self, results_dir):
198    pass
199
200  def ReadAutotestOutput(self, destination):
201    pass
202
203  def StorePerfResults(self, perf):
204    pass
205
206  def ReadPerfResults(self):
207    return PerfProcessor.PerfResults("", "")
208