results_cache.py revision b47bff4d3336c5fe5593a95963c0f3dc20a02f68 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import getpass
8import glob
9import hashlib
10import os
11import pickle
12import re
13import tempfile
14
15from utils import command_executer
16from utils import misc
17
18from image_checksummer import ImageChecksummer
19
20SCRATCH_BASE = "/home/%s/cros_scratch"
21SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
22RESULTS_FILE = "results.txt"
23MACHINE_FILE = "machine.txt"
24AUTOTEST_TARBALL = "autotest.tbz2"
25PERF_RESULTS_FILE = "perf-results.txt"
26
27
28class Result(object):
29  """ This class manages what exactly is stored inside the cache without knowing
30  what the key of the cache is. For runs with perf, it stores perf.data,
31  perf.report, etc. The key generation is handled by the ResultsCache class.
32  """
33
34  def __init__(self, logger, label):
35    self._chromeos_root = label.chromeos_root
36    self._logger = logger
37    self._ce = command_executer.GetCommandExecuter(self._logger)
38    self._temp_dir = None
39    self.label = label
40    self.results_dir = None
41    self.perf_data_files = []
42    self.perf_report_files = []
43
44  def _CopyFilesTo(self, dest_dir, files_to_copy):
45    file_index = 0
46    for file_to_copy in files_to_copy:
47      if not os.path.isdir(dest_dir):
48        command = "mkdir -p %s" % dest_dir
49        self._ce.RunCommand(command)
50      dest_file = os.path.join(dest_dir,
51                               ("%s.%s" % (os.path.basename(file_to_copy),
52                                           file_index)))
53      ret = self._ce.CopyFiles(file_to_copy,
54                               dest_file,
55                               recursive=False)
56      if ret:
57        raise Exception("Could not copy results file: %s" % file_to_copy)
58
59  def CopyResultsTo(self, dest_dir):
60    self._CopyFilesTo(dest_dir, self.perf_data_files)
61    self._CopyFilesTo(dest_dir, self.perf_report_files)
62
63  def _GetKeyvals(self):
64    results_in_chroot = os.path.join(self._chromeos_root,
65                                     "chroot", "tmp")
66    if not self._temp_dir:
67      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
68      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
69      self._ce.RunCommand(command)
70
71    command = ("python generate_test_report --no-color --csv %s" %
72               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
73    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
74                                            command,
75                                            return_output=True)
76    keyvals_dict = {}
77    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
78                                                 self._temp_dir)
79    for line in out.splitlines():
80      tokens = re.split("=|,", line)
81      key = tokens[-2]
82      if key.startswith(tmp_dir_in_chroot):
83        key = key[len(tmp_dir_in_chroot) + 1:]
84      value = tokens[-1]
85      keyvals_dict[key] = value
86
87    return keyvals_dict
88
89  def _GetResultsDir(self):
90    mo = re.search(r"Results placed in (\S+)", self.out)
91    if mo:
92      result = mo.group(1)
93      return result
94    raise Exception("Could not find results directory.")
95
96  def _FindFilesInResultsDir(self, find_args):
97    if not self.results_dir:
98      return None
99
100    command = "find %s %s" % (self.results_dir,
101                              find_args)
102    ret, out, _ = self._ce.RunCommand(command, return_output=True)
103    if ret:
104      raise Exception("Could not run find command!")
105    return out
106
107  def _GetPerfDataFiles(self):
108    return self._FindFilesInResultsDir("-name perf.data").splitlines()
109
110  def _GetPerfReportFiles(self):
111    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
112
113  def _GeneratePerfReportFiles(self):
114    perf_report_files = []
115    for perf_data_file in self.perf_data_files:
116      # Generate a perf.report and store it side-by-side with the perf.data
117      # file.
118      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
119                                                       perf_data_file)
120      perf_report_file = "%s.report" % perf_data_file
121      if os.path.exists(perf_report_file):
122        raise Exception("Perf report file already exists: %s" %
123                        perf_report_file)
124      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
125                                                         perf_report_file)
126      command = ("/usr/sbin/perf report "
127                 "-n "
128                 "--symfs /build/%s "
129                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
130                 "--kallsyms /build/%s/boot/System.map-* "
131                 "-i %s --stdio "
132                 "> %s" %
133                 (self._board,
134                  self._board,
135                  self._board,
136                  chroot_perf_data_file,
137                  chroot_perf_report_file))
138      self._ce.ChrootRunCommand(self._chromeos_root,
139                                command)
140
141      # Add a keyval to the dictionary for the events captured.
142      perf_report_files.append(
143          misc.GetOutsideChrootPath(self._chromeos_root,
144                                    chroot_perf_report_file))
145    return perf_report_files
146
147  def _GatherPerfResults(self):
148    report_id = 0
149    for perf_report_file in self.perf_report_files:
150      with open(perf_report_file, "r") as f:
151        report_contents = f.read()
152        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
153          num_events = group[0]
154          event_name = group[1]
155          key = "perf_%s_%s" % (report_id, event_name)
156          value = str(misc.UnitToNumber(num_events))
157          self.keyvals[key] = value
158
159  def _PopulateFromRun(self, out, err, retval):
160    self._board = self.label.board
161    self.out = out
162    self.err = err
163    self.retval = retval
164    self.chroot_results_dir = self._GetResultsDir()
165    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
166                                                 self.chroot_results_dir)
167    self.perf_data_files = self._GetPerfDataFiles()
168    # Include all perf.report data in table.
169    self.perf_report_files = self._GeneratePerfReportFiles()
170    # TODO(asharif): Do something similar with perf stat.
171
172    # Grab keyvals from the directory.
173    self._ProcessResults()
174
175  def _ProcessResults(self):
176    # Note that this function doesn't know anything about whether there is a
177    # cache hit or miss. It should process results agnostic of the cache hit
178    # state.
179    self.keyvals = self._GetKeyvals()
180    self.keyvals["retval"] = self.retval
181    # Generate report from all perf.data files.
182    # Now parse all perf report files and include them in keyvals.
183    self._GatherPerfResults()
184
185  def _PopulateFromCacheDir(self, cache_dir):
186    # Read in everything from the cache directory.
187    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
188      self.out = pickle.load(f)
189      self.err = pickle.load(f)
190      self.retval = pickle.load(f)
191
192    # Untar the tarball to a temporary directory
193    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
194                                                       "chroot", "tmp"))
195
196    command = ("cd %s && tar xf %s" %
197               (self._temp_dir,
198                os.path.join(cache_dir, AUTOTEST_TARBALL)))
199    ret = self._ce.RunCommand(command)
200    if ret:
201      raise Exception("Could not untar cached tarball")
202    self.results_dir = self._temp_dir
203    self.perf_data_files = self._GetPerfDataFiles()
204    self.perf_report_files = self._GetPerfReportFiles()
205    self._ProcessResults()
206
207  def CleanUp(self, rm_chroot_tmp):
208    if rm_chroot_tmp and self.results_dir:
209      command = "rm -rf %s" % self.results_dir
210      self._ce.RunCommand(command)
211    if self._temp_dir:
212      command = "rm -rf %s" % self._temp_dir
213      self._ce.RunCommand(command)
214
215  def StoreToCacheDir(self, cache_dir, machine_manager):
216    # Create the dir if it doesn't exist.
217    temp_dir = tempfile.mkdtemp()
218
219    # Store to the temp directory.
220    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
221      pickle.dump(self.out, f)
222      pickle.dump(self.err, f)
223      pickle.dump(self.retval, f)
224
225    if self.results_dir:
226      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
227      command = ("cd %s && "
228                 "tar "
229                 "--exclude=var/spool "
230                 "--exclude=var/log "
231                 "-cjf %s ." % (self.results_dir, tarball))
232      ret = self._ce.RunCommand(command)
233      if ret:
234        raise Exception("Couldn't store autotest output directory.")
235    # Store machine info.
236    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
237    # this function.
238    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
239      f.write(machine_manager.machine_checksum_string[self.label.name])
240
241    if os.path.exists(cache_dir):
242      command = "rm -rf {0}".format(cache_dir)
243      self._ce.RunCommand(command)
244
245    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
246    command += "chmod g+x {0} && ".format(temp_dir)
247    command += "mv {0} {1}".format(temp_dir, cache_dir)
248    ret = self._ce.RunCommand(command)
249    if ret:
250      command = "rm -rf {0}".format(temp_dir)
251      self._ce.RunCommand(command)
252      raise Exception("Could not move dir %s to dir %s" %
253                      (temp_dir, cache_dir))
254
255  @classmethod
256  def CreateFromRun(cls, logger, label, out, err, retval, suite="pyauto"):
257    if suite == "telemetry":
258      result = TelemetryResult(logger, label)
259    else:
260      result = cls(logger, label)
261    result._PopulateFromRun(out, err, retval)
262    return result
263
264  @classmethod
265  def CreateFromCacheHit(cls, logger, label, cache_dir,
266                         suite="pyauto"):
267    if suite == "telemetry":
268      result = TelemetryResult(logger, label)
269    else:
270      result = cls(logger, label)
271    try:
272      result._PopulateFromCacheDir(cache_dir)
273
274    except Exception as e:
275      logger.LogError("Exception while using cache: %s" % e)
276      return None
277    return result
278
279class TelemetryResult(Result):
280
281  def __init__(self, logger, label):
282    super(TelemetryResult, self).__init__(logger, label)
283
284  def _PopulateFromRun(self, out, err, retval):
285    self.out = out
286    self.err = err
287    self.retval = retval
288
289    self._ProcessResults()
290
291  def _ProcessResults(self):
292    # The output is:
293    # url,average_commit_time (ms),...
294    # www.google.com,33.4,21.2,...
295    # We need to convert to this format:
296    # {"www.google.com:average_commit_time (ms)": "33.4",
297    #  "www.google.com:...": "21.2"}
298    # Added note:  Occasionally the output comes back
299    # with "JSON.stringify(window.automation.GetResults())" on
300    # the first line, and then the rest of the output as
301    # described above.
302
303    lines = self.out.splitlines()
304    self.keyvals = {}
305
306    if lines:
307      if lines[0].startswith("JSON.stringify"):
308        lines = lines[1:]
309
310    if not lines:
311      return
312    labels = lines[0].split(",")
313    for line in lines[1:]:
314      fields = line.split(",")
315      if (len(fields) != len(labels)):
316        continue
317      for i in range(1, len(labels)):
318        key = "%s %s" % (fields[0], labels[i])
319        value = fields[i]
320        self.keyvals[key] = value
321    self.keyvals["retval"] = self.retval
322
323  def _PopulateFromCacheDir(self, cache_dir):
324    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
325      self.out = pickle.load(f)
326      self.err = pickle.load(f)
327      self.retval = pickle.load(f)
328    self._ProcessResults()
329
330
331class CacheConditions(object):
332  # Cache hit only if the result file exists.
333  CACHE_FILE_EXISTS = 0
334
335  # Cache hit if the checksum of cpuinfo and totalmem of
336  # the cached result and the new run match.
337  MACHINES_MATCH = 1
338
339  # Cache hit if the image checksum of the cached result and the new run match.
340  CHECKSUMS_MATCH = 2
341
342  # Cache hit only if the cached result was successful
343  RUN_SUCCEEDED = 3
344
345  # Never a cache hit.
346  FALSE = 4
347
348  # Cache hit if the image path matches the cached image path.
349  IMAGE_PATH_MATCH = 5
350
351  # Cache hit if the uuid of hard disk mataches the cached one
352
353  SAME_MACHINE_MATCH = 6
354
355
356class ResultsCache(object):
357  """ This class manages the key of the cached runs without worrying about what
358  is exactly stored (value). The value generation is handled by the Results
359  class.
360  """
361  CACHE_VERSION = 6
362
363  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
364           test_args, machine_manager, board, cache_conditions,
365           logger_to_use, label, share_users, suite):
366    self.chromeos_image = chromeos_image
367    self.chromeos_root = chromeos_root
368    self.test_name = test_name
369    self.iteration = iteration
370    self.test_args = test_args,
371    self.board = board
372    self.cache_conditions = cache_conditions
373    self.machine_manager = machine_manager
374    self._logger = logger_to_use
375    self._ce = command_executer.GetCommandExecuter(self._logger)
376    self.label = label
377    self.share_users = share_users
378    self.suite = suite
379
380  def _GetCacheDirForRead(self):
381    matching_dirs = []
382    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
383      matching_dirs += glob.glob(glob_path)
384
385    if matching_dirs:
386      # Cache file found.
387      return matching_dirs[0]
388    else:
389      return None
390
391  def _GetCacheDirForWrite(self):
392    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
393
394  def _FormCacheDir(self, list_of_strings):
395    cache_key = " ".join(list_of_strings)
396    cache_dir = misc.GetFilenameFromString(cache_key)
397    if self.label.cache_dir:
398      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
399      cache_path = [os.path.join(cache_home, cache_dir)]
400    else:
401      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
402
403    for i in [x.strip() for x in self.share_users.split(",")]:
404      path = SCRATCH_BASE % i
405      cache_path.append(os.path.join(path, cache_dir))
406
407    return cache_path
408
409  def _GetCacheKeyList(self, read):
410    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
411      machine_checksum = "*"
412    else:
413      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
414    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
415      checksum = "*"
416    else:
417      checksum = ImageChecksummer().Checksum(self.label)
418
419    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
420      image_path_checksum = "*"
421    else:
422      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
423
424    machine_id_checksum = ""
425    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
426      machine_id_checksum = "*"
427    else:
428      for machine in self.machine_manager.GetMachines(self.label):
429        if machine.name == self.label.remote[0]:
430          machine_id_checksum = machine.machine_id_checksum
431          break
432
433    test_args_checksum = hashlib.md5(
434        "".join(self.test_args)).hexdigest()
435    return (image_path_checksum,
436            self.test_name, str(self.iteration),
437            test_args_checksum,
438            checksum,
439            machine_checksum,
440            machine_id_checksum,
441            str(self.CACHE_VERSION))
442
443  def ReadResult(self):
444    if CacheConditions.FALSE in self.cache_conditions:
445      return None
446    cache_dir = self._GetCacheDirForRead()
447
448    if not cache_dir:
449      return None
450
451    if not os.path.isdir(cache_dir):
452      return None
453
454    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
455    result = Result.CreateFromCacheHit(self._logger,
456                                       self.label,
457                                       cache_dir,
458                                       self.suite)
459    if not result:
460      return None
461
462    if (result.retval == 0 or
463        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
464      return result
465
466    return None
467
468  def StoreResult(self, result):
469    cache_dir = self._GetCacheDirForWrite()
470    result.StoreToCacheDir(cache_dir, self.machine_manager)
471
472
473class MockResultsCache(ResultsCache):
474  def Init(self, *args):
475    pass
476
477  def ReadResult(self):
478    return None
479
480  def StoreResult(self, result):
481    pass
482
483
484class MockResult(Result):
485  def _PopulateFromRun(self, out, err, retval):
486    self.out = out
487    self.err = err
488    self.retval = retval
489
490