results_cache.py revision 4f10d39a499ad872c22402aa763037fc3351f399 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import getpass
8import glob
9import hashlib
10import os
11import pickle
12import re
13import tempfile
14
15from utils import command_executer
16from utils import misc
17
18from image_checksummer import ImageChecksummer
19
20SCRATCH_BASE = "/home/%s/cros_scratch"
21SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
22RESULTS_FILE = "results.txt"
23MACHINE_FILE = "machine.txt"
24AUTOTEST_TARBALL = "autotest.tbz2"
25PERF_RESULTS_FILE = "perf-results.txt"
26
27
28class Result(object):
29  """ This class manages what exactly is stored inside the cache without knowing
30  what the key of the cache is. For runs with perf, it stores perf.data,
31  perf.report, etc. The key generation is handled by the ResultsCache class.
32  """
33
34  def __init__(self, logger, label):
35    self._chromeos_root = label.chromeos_root
36    self._logger = logger
37    self._ce = command_executer.GetCommandExecuter(self._logger)
38    self._temp_dir = None
39    self.label = label
40    self.results_dir = None
41    self.perf_data_files = []
42    self.perf_report_files = []
43
44  def _CopyFilesTo(self, dest_dir, files_to_copy):
45    file_index = 0
46    for file_to_copy in files_to_copy:
47      if not os.path.isdir(dest_dir):
48        command = "mkdir -p %s" % dest_dir
49        self._ce.RunCommand(command)
50      dest_file = os.path.join(dest_dir,
51                               ("%s.%s" % (os.path.basename(file_to_copy),
52                                           file_index)))
53      ret = self._ce.CopyFiles(file_to_copy,
54                               dest_file,
55                               recursive=False)
56      if ret:
57        raise Exception("Could not copy results file: %s" % file_to_copy)
58
59  def CopyResultsTo(self, dest_dir):
60    self._CopyFilesTo(dest_dir, self.perf_data_files)
61    self._CopyFilesTo(dest_dir, self.perf_report_files)
62
63  def _GetKeyvals(self):
64    results_in_chroot = os.path.join(self._chromeos_root,
65                                     "chroot", "tmp")
66    if not self._temp_dir:
67      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
68      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
69      self._ce.RunCommand(command)
70
71    command = ("python generate_test_report --no-color --csv %s" %
72               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
73    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
74                                            command,
75                                            return_output=True)
76    keyvals_dict = {}
77    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
78                                                 self._temp_dir)
79    for line in out.splitlines():
80      tokens = re.split("=|,", line)
81      key = tokens[-2]
82      if key.startswith(tmp_dir_in_chroot):
83        key = key[len(tmp_dir_in_chroot) + 1:]
84      value = tokens[-1]
85      keyvals_dict[key] = value
86
87    return keyvals_dict
88
89  def _GetResultsDir(self):
90    mo = re.search(r"Results placed in (\S+)", self.out)
91    if mo:
92      result = mo.group(1)
93      return result
94    raise Exception("Could not find results directory.")
95
96  def _FindFilesInResultsDir(self, find_args):
97    if not self.results_dir:
98      return None
99    command = "find %s %s" % (self.results_dir,
100                              find_args)
101    ret, out, _ = self._ce.RunCommand(command, return_output=True)
102    if ret:
103      raise Exception("Could not run find command!")
104    return out
105
106  def _GetPerfDataFiles(self):
107    return self._FindFilesInResultsDir("-name perf.data").splitlines()
108
109  def _GetPerfReportFiles(self):
110    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
111
112  def _GeneratePerfReportFiles(self):
113    perf_report_files = []
114    for perf_data_file in self.perf_data_files:
115      # Generate a perf.report and store it side-by-side with the perf.data
116      # file.
117      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
118                                                       perf_data_file)
119      perf_report_file = "%s.report" % perf_data_file
120      if os.path.exists(perf_report_file):
121        raise Exception("Perf report file already exists: %s" %
122                        perf_report_file)
123      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
124                                                         perf_report_file)
125      command = ("/usr/sbin/perf report "
126                 "-n "
127                 "--symfs /build/%s "
128                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
129                 "--kallsyms /build/%s/boot/System.map-* "
130                 "-i %s --stdio "
131                 "> %s" %
132                 (self._board,
133                  self._board,
134                  self._board,
135                  chroot_perf_data_file,
136                  chroot_perf_report_file))
137      self._ce.ChrootRunCommand(self._chromeos_root,
138                                command)
139
140      # Add a keyval to the dictionary for the events captured.
141      perf_report_files.append(
142          misc.GetOutsideChrootPath(self._chromeos_root,
143                                    chroot_perf_report_file))
144    return perf_report_files
145
146  def _GatherPerfResults(self):
147    report_id = 0
148    for perf_report_file in self.perf_report_files:
149      with open(perf_report_file, "r") as f:
150        report_contents = f.read()
151        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
152          num_events = group[0]
153          event_name = group[1]
154          key = "perf_%s_%s" % (report_id, event_name)
155          value = str(misc.UnitToNumber(num_events))
156          self.keyvals[key] = value
157
158  def _PopulateFromRun(self, out, err, retval):
159    self._board = self.label.board
160    self.out = out
161    self.err = err
162    self.retval = retval
163    self.chroot_results_dir = self._GetResultsDir()
164    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
165                                                 self.chroot_results_dir)
166    self.perf_data_files = self._GetPerfDataFiles()
167    # Include all perf.report data in table.
168    self.perf_report_files = self._GeneratePerfReportFiles()
169    # TODO(asharif): Do something similar with perf stat.
170
171    # Grab keyvals from the directory.
172    self._ProcessResults()
173
174  def _ProcessResults(self):
175    # Note that this function doesn't know anything about whether there is a
176    # cache hit or miss. It should process results agnostic of the cache hit
177    # state.
178    self.keyvals = self._GetKeyvals()
179    self.keyvals["retval"] = self.retval
180    # Generate report from all perf.data files.
181    # Now parse all perf report files and include them in keyvals.
182    self._GatherPerfResults()
183
184  def _PopulateFromCacheDir(self, cache_dir):
185    # Read in everything from the cache directory.
186    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
187      self.out = pickle.load(f)
188      self.err = pickle.load(f)
189      self.retval = pickle.load(f)
190
191    # Untar the tarball to a temporary directory
192    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
193                                                       "chroot", "tmp"))
194
195    command = ("cd %s && tar xf %s" %
196               (self._temp_dir,
197                os.path.join(cache_dir, AUTOTEST_TARBALL)))
198    ret = self._ce.RunCommand(command)
199    if ret:
200      raise Exception("Could not untar cached tarball")
201    self.results_dir = self._temp_dir
202    self.perf_data_files = self._GetPerfDataFiles()
203    self.perf_report_files = self._GetPerfReportFiles()
204    self._ProcessResults()
205
206  def CleanUp(self, rm_chroot_tmp):
207    if rm_chroot_tmp and self.results_dir:
208      command = "rm -rf %s" % self.results_dir
209      self._ce.RunCommand(command)
210    if self._temp_dir:
211      command = "rm -rf %s" % self._temp_dir
212      self._ce.RunCommand(command)
213
214  def StoreToCacheDir(self, cache_dir, machine_manager):
215    # Create the dir if it doesn't exist.
216    temp_dir = tempfile.mkdtemp()
217
218    # Store to the temp directory.
219    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
220      pickle.dump(self.out, f)
221      pickle.dump(self.err, f)
222      pickle.dump(self.retval, f)
223
224    if self.results_dir:
225      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
226      command = ("cd %s && "
227                 "tar "
228                 "--exclude=var/spool "
229                 "--exclude=var/log "
230                 "-cjf %s ." % (self.results_dir, tarball))
231      ret = self._ce.RunCommand(command)
232      if ret:
233        raise Exception("Couldn't store autotest output directory.")
234    # Store machine info.
235    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
236    # this function.
237    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
238      f.write(machine_manager.machine_checksum_string[self.label.name])
239
240    if os.path.exists(cache_dir):
241      command = "rm -rf {0}".format(cache_dir)
242      self._ce.RunCommand(command)
243
244    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
245    command += "chmod g+x {0} && ".format(temp_dir)
246    command += "mv {0} {1}".format(temp_dir, cache_dir)
247    ret = self._ce.RunCommand(command)
248    if ret:
249      command = "rm -rf {0}".format(temp_dir)
250      self._ce.RunCommand(command)
251      raise Exception("Could not move dir %s to dir %s" %
252                      (temp_dir, cache_dir))
253
254  @classmethod
255  def CreateFromRun(cls, logger, label, out, err, retval, suite="pyauto"):
256    if suite == "telemetry":
257      result = TelemetryResult(logger, label)
258    else:
259      result = cls(logger, label)
260    result._PopulateFromRun(out, err, retval)
261    return result
262
263  @classmethod
264  def CreateFromCacheHit(cls, logger, label, cache_dir,
265                         suite="pyauto"):
266    if suite == "telemetry":
267      result = TelemetryResult(logger, label)
268    else:
269      result = cls(logger, label)
270    try:
271      result._PopulateFromCacheDir(cache_dir)
272
273    except Exception as e:
274      logger.LogError("Exception while using cache: %s" % e)
275      return None
276    return result
277
278class TelemetryResult(Result):
279
280  def __init__(self, logger, label):
281    super(TelemetryResult, self).__init__(logger, label)
282
283  def _PopulateFromRun(self, out, err, retval):
284    self.out = out
285    self.err = err
286    self.retval = retval
287
288    self._ProcessResults()
289
290  def _ProcessResults(self):
291    # The output is:
292    # url,average_commit_time (ms),...
293    # www.google.com,33.4,21.2,...
294    # We need to convert to this format:
295    # {"www.google.com:average_commit_time (ms)": "33.4",
296    #  "www.google.com:...": "21.2"}
297    # Added note:  Occasionally the output comes back
298    # with "JSON.stringify(window.automation.GetResults())" on
299    # the first line, and then the rest of the output as
300    # described above.
301
302    lines = self.out.splitlines()
303    self.keyvals = {}
304
305    if lines:
306      if lines[0].startswith("JSON.stringify"):
307        lines = lines[1:]
308
309    if not lines:
310      return
311    labels = lines[0].split(",")
312    for line in lines[1:]:
313      fields = line.split(",")
314      if (len(fields) != len(labels)):
315        continue
316      for i in range(1, len(labels)):
317        key = "%s %s" % (fields[0], labels[i])
318        value = fields[i]
319        self.keyvals[key] = value
320    self.keyvals["retval"] = self.retval
321
322  def _PopulateFromCacheDir(self, cache_dir):
323    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
324      self.out = pickle.load(f)
325      self.err = pickle.load(f)
326      self.retval = pickle.load(f)
327    self._ProcessResults()
328
329
330class CacheConditions(object):
331  # Cache hit only if the result file exists.
332  CACHE_FILE_EXISTS = 0
333
334  # Cache hit if the checksum of cpuinfo and totalmem of
335  # the cached result and the new run match.
336  MACHINES_MATCH = 1
337
338  # Cache hit if the image checksum of the cached result and the new run match.
339  CHECKSUMS_MATCH = 2
340
341  # Cache hit only if the cached result was successful
342  RUN_SUCCEEDED = 3
343
344  # Never a cache hit.
345  FALSE = 4
346
347  # Cache hit if the image path matches the cached image path.
348  IMAGE_PATH_MATCH = 5
349
350  # Cache hit if the uuid of hard disk mataches the cached one
351
352  SAME_MACHINE_MATCH = 6
353
354
355class ResultsCache(object):
356  """ This class manages the key of the cached runs without worrying about what
357  is exactly stored (value). The value generation is handled by the Results
358  class.
359  """
360  CACHE_VERSION = 6
361
362  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
363           test_args, machine_manager, board, cache_conditions,
364           logger_to_use, label, share_users, suite):
365    self.chromeos_image = chromeos_image
366    self.chromeos_root = chromeos_root
367    self.test_name = test_name
368    self.iteration = iteration
369    self.test_args = test_args,
370    self.board = board
371    self.cache_conditions = cache_conditions
372    self.machine_manager = machine_manager
373    self._logger = logger_to_use
374    self._ce = command_executer.GetCommandExecuter(self._logger)
375    self.label = label
376    self.share_users = share_users
377    self.suite = suite
378
379  def _GetCacheDirForRead(self):
380    matching_dirs = []
381    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
382      matching_dirs += glob.glob(glob_path)
383
384    if matching_dirs:
385      # Cache file found.
386      return matching_dirs[0]
387    else:
388      return None
389
390  def _GetCacheDirForWrite(self):
391    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
392
393  def _FormCacheDir(self, list_of_strings):
394    cache_key = " ".join(list_of_strings)
395    cache_dir = misc.GetFilenameFromString(cache_key)
396    if self.label.cache_dir:
397      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
398      cache_path = [os.path.join(cache_home, cache_dir)]
399    else:
400      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
401
402    for i in [x.strip() for x in self.share_users.split(",")]:
403      path = SCRATCH_BASE % i
404      cache_path.append(os.path.join(path, cache_dir))
405
406    return cache_path
407
408  def _GetCacheKeyList(self, read):
409    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
410      machine_checksum = "*"
411    else:
412      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
413    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
414      checksum = "*"
415    else:
416      checksum = ImageChecksummer().Checksum(self.label)
417
418    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
419      image_path_checksum = "*"
420    else:
421      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
422
423    machine_id_checksum = ""
424    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
425      machine_id_checksum = "*"
426    else:
427      for machine in self.machine_manager.GetMachines(self.label):
428        if machine.name == self.label.remote[0]:
429          machine_id_checksum = machine.machine_id_checksum
430          break
431
432    test_args_checksum = hashlib.md5(
433        "".join(self.test_args)).hexdigest()
434    return (image_path_checksum,
435            self.test_name, str(self.iteration),
436            test_args_checksum,
437            checksum,
438            machine_checksum,
439            machine_id_checksum,
440            str(self.CACHE_VERSION))
441
442  def ReadResult(self):
443    if CacheConditions.FALSE in self.cache_conditions:
444      return None
445    cache_dir = self._GetCacheDirForRead()
446
447    if not cache_dir:
448      return None
449
450    if not os.path.isdir(cache_dir):
451      return None
452
453    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
454    result = Result.CreateFromCacheHit(self._logger,
455                                       self.label,
456                                       cache_dir,
457                                       self.suite)
458    if not result:
459      return None
460
461    if (result.retval == 0 or
462        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
463      return result
464
465    return None
466
467  def StoreResult(self, result):
468    cache_dir = self._GetCacheDirForWrite()
469    result.StoreToCacheDir(cache_dir, self.machine_manager)
470
471
472class MockResultsCache(ResultsCache):
473  def Init(self, *args):
474    pass
475
476  def ReadResult(self):
477    return None
478
479  def StoreResult(self, result):
480    pass
481
482
483class MockResult(Result):
484  def _PopulateFromRun(self, out, err, retval):
485    self.out = out
486    self.err = err
487    self.retval = retval
488
489