results_cache.py revision d80a960b62d89bc6f83d2f5fd4d599408f2ec7f9 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to deal with result cache."""
8
9import getpass
10import glob
11import hashlib
12import os
13import pickle
14import re
15import tempfile
16import json
17
18from utils import command_executer
19from utils import misc
20
21from image_checksummer import ImageChecksummer
22
23SCRATCH_BASE = "/home/%s/cros_scratch"
24SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
25RESULTS_FILE = "results.txt"
26MACHINE_FILE = "machine.txt"
27AUTOTEST_TARBALL = "autotest.tbz2"
28PERF_RESULTS_FILE = "perf-results.txt"
29
30
31class Result(object):
32  """ This class manages what exactly is stored inside the cache without knowing
33  what the key of the cache is. For runs with perf, it stores perf.data,
34  perf.report, etc. The key generation is handled by the ResultsCache class.
35  """
36
37  def __init__(self, logger, label):
38    self._chromeos_root = label.chromeos_root
39    self._logger = logger
40    self._ce = command_executer.GetCommandExecuter(self._logger)
41    self._temp_dir = None
42    self.label = label
43    self.results_dir = None
44    self.perf_data_files = []
45    self.perf_report_files = []
46
47  def _CopyFilesTo(self, dest_dir, files_to_copy):
48    file_index = 0
49    for file_to_copy in files_to_copy:
50      if not os.path.isdir(dest_dir):
51        command = "mkdir -p %s" % dest_dir
52        self._ce.RunCommand(command)
53      dest_file = os.path.join(dest_dir,
54                               ("%s.%s" % (os.path.basename(file_to_copy),
55                                           file_index)))
56      ret = self._ce.CopyFiles(file_to_copy,
57                               dest_file,
58                               recursive=False)
59      if ret:
60        raise Exception("Could not copy results file: %s" % file_to_copy)
61
62  def CopyResultsTo(self, dest_dir):
63    self._CopyFilesTo(dest_dir, self.perf_data_files)
64    self._CopyFilesTo(dest_dir, self.perf_report_files)
65
66  def _GetNewKeyvals(self, keyvals_dict):
67    results_files = self._GetDataMeasurementsFiles()
68    for f in results_files:
69      # Make sure we can find the results file
70      if os.path.exists(f):
71        data_filename = f
72      else:
73        # Otherwise get the base filename and create the correct
74        # path for it.
75        f_dir, f_base = misc.GetRoot(f)
76        data_filename = os.path.join(self._chromeos_root, "/tmp",
77                                     self._temp_dir, f_base)
78      if os.path.exists(data_filename):
79        with open(data_filename, "r") as data_file:
80          lines = data_file.readlines()
81          for line in lines:
82            tmp_dict = json.loads(line)
83            key = tmp_dict['description']
84            val = tmp_dict['value']
85            keyvals_dict[key] = val
86
87    return keyvals_dict
88
89
90  def _GetKeyvals(self):
91    results_in_chroot = os.path.join(self._chromeos_root,
92                                     "chroot", "tmp")
93    if not self._temp_dir:
94      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
95      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
96      self._ce.RunCommand(command)
97
98    command = ("python generate_test_report --no-color --csv %s" %
99               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
100    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
101                                            command,
102                                            return_output=True)
103    keyvals_dict = {}
104    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
105                                                 self._temp_dir)
106    for line in out.splitlines():
107      tokens = re.split("=|,", line)
108      key = tokens[-2]
109      if key.startswith(tmp_dir_in_chroot):
110        key = key[len(tmp_dir_in_chroot) + 1:]
111      value = tokens[-1]
112      keyvals_dict[key] = value
113
114    # Check to see if there is a perf_measurements file and get the
115    # data from it if so.
116    return self._GetNewKeyvals(keyvals_dict)
117
118  def _GetResultsDir(self):
119    mo = re.search(r"Results placed in (\S+)", self.out)
120    if mo:
121      result = mo.group(1)
122      return result
123    raise Exception("Could not find results directory.")
124
125  def _FindFilesInResultsDir(self, find_args):
126    if not self.results_dir:
127      return None
128
129    command = "find %s %s" % (self.results_dir,
130                              find_args)
131    ret, out, _ = self._ce.RunCommand(command, return_output=True)
132    if ret:
133      raise Exception("Could not run find command!")
134    return out
135
136  def _GetPerfDataFiles(self):
137    return self._FindFilesInResultsDir("-name perf.data").splitlines()
138
139  def _GetPerfReportFiles(self):
140    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
141
142  def _GetDataMeasurementsFiles(self):
143    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()
144
145  def _GeneratePerfReportFiles(self):
146    perf_report_files = []
147    for perf_data_file in self.perf_data_files:
148      # Generate a perf.report and store it side-by-side with the perf.data
149      # file.
150      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
151                                                       perf_data_file)
152      perf_report_file = "%s.report" % perf_data_file
153      if os.path.exists(perf_report_file):
154        raise Exception("Perf report file already exists: %s" %
155                        perf_report_file)
156      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
157                                                         perf_report_file)
158      command = ("/usr/sbin/perf report "
159                 "-n "
160                 "--symfs /build/%s "
161                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
162                 "--kallsyms /build/%s/boot/System.map-* "
163                 "-i %s --stdio "
164                 "> %s" %
165                 (self._board,
166                  self._board,
167                  self._board,
168                  chroot_perf_data_file,
169                  chroot_perf_report_file))
170      self._ce.ChrootRunCommand(self._chromeos_root,
171                                command)
172
173      # Add a keyval to the dictionary for the events captured.
174      perf_report_files.append(
175          misc.GetOutsideChrootPath(self._chromeos_root,
176                                    chroot_perf_report_file))
177    return perf_report_files
178
179  def _GatherPerfResults(self):
180    report_id = 0
181    for perf_report_file in self.perf_report_files:
182      with open(perf_report_file, "r") as f:
183        report_contents = f.read()
184        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
185          num_events = group[0]
186          event_name = group[1]
187          key = "perf_%s_%s" % (report_id, event_name)
188          value = str(misc.UnitToNumber(num_events))
189          self.keyvals[key] = value
190
191  def _PopulateFromRun(self, out, err, retval):
192    self._board = self.label.board
193    self.out = out
194    self.err = err
195    self.retval = retval
196    self.chroot_results_dir = self._GetResultsDir()
197    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
198                                                 self.chroot_results_dir)
199    self.perf_data_files = self._GetPerfDataFiles()
200    # Include all perf.report data in table.
201    self.perf_report_files = self._GeneratePerfReportFiles()
202    # TODO(asharif): Do something similar with perf stat.
203
204    # Grab keyvals from the directory.
205    self._ProcessResults()
206
207  def _ProcessResults(self):
208    # Note that this function doesn't know anything about whether there is a
209    # cache hit or miss. It should process results agnostic of the cache hit
210    # state.
211    self.keyvals = self._GetKeyvals()
212    self.keyvals["retval"] = self.retval
213    # Generate report from all perf.data files.
214    # Now parse all perf report files and include them in keyvals.
215    self._GatherPerfResults()
216
217  def _PopulateFromCacheDir(self, cache_dir):
218    # Read in everything from the cache directory.
219    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
220      self.out = pickle.load(f)
221      self.err = pickle.load(f)
222      self.retval = pickle.load(f)
223
224    # Untar the tarball to a temporary directory
225    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
226                                                       "chroot", "tmp"))
227
228    command = ("cd %s && tar xf %s" %
229               (self._temp_dir,
230                os.path.join(cache_dir, AUTOTEST_TARBALL)))
231    ret = self._ce.RunCommand(command)
232    if ret:
233      raise Exception("Could not untar cached tarball")
234    self.results_dir = self._temp_dir
235    self.perf_data_files = self._GetPerfDataFiles()
236    self.perf_report_files = self._GetPerfReportFiles()
237    self._ProcessResults()
238
239  def CleanUp(self, rm_chroot_tmp):
240    if rm_chroot_tmp and self.results_dir:
241      command = "rm -rf %s" % self.results_dir
242      self._ce.RunCommand(command)
243    if self._temp_dir:
244      command = "rm -rf %s" % self._temp_dir
245      self._ce.RunCommand(command)
246
247  def StoreToCacheDir(self, cache_dir, machine_manager):
248    # Create the dir if it doesn't exist.
249    temp_dir = tempfile.mkdtemp()
250
251    # Store to the temp directory.
252    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
253      pickle.dump(self.out, f)
254      pickle.dump(self.err, f)
255      pickle.dump(self.retval, f)
256
257    if self.results_dir:
258      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
259      command = ("cd %s && "
260                 "tar "
261                 "--exclude=var/spool "
262                 "--exclude=var/log "
263                 "-cjf %s ." % (self.results_dir, tarball))
264      ret = self._ce.RunCommand(command)
265      if ret:
266        raise Exception("Couldn't store autotest output directory.")
267    # Store machine info.
268    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
269    # this function.
270    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
271      f.write(machine_manager.machine_checksum_string[self.label.name])
272
273    if os.path.exists(cache_dir):
274      command = "rm -rf {0}".format(cache_dir)
275      self._ce.RunCommand(command)
276
277    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
278    command += "chmod g+x {0} && ".format(temp_dir)
279    command += "mv {0} {1}".format(temp_dir, cache_dir)
280    ret = self._ce.RunCommand(command)
281    if ret:
282      command = "rm -rf {0}".format(temp_dir)
283      self._ce.RunCommand(command)
284      raise Exception("Could not move dir %s to dir %s" %
285                      (temp_dir, cache_dir))
286
287  @classmethod
288  def CreateFromRun(cls, logger, label, out, err, retval, suite="pyauto"):
289    if suite == "telemetry":
290      result = TelemetryResult(logger, label)
291    else:
292      result = cls(logger, label)
293    result._PopulateFromRun(out, err, retval)
294    return result
295
296  @classmethod
297  def CreateFromCacheHit(cls, logger, label, cache_dir,
298                         suite="pyauto"):
299    if suite == "telemetry":
300      result = TelemetryResult(logger, label)
301    else:
302      result = cls(logger, label)
303    try:
304      result._PopulateFromCacheDir(cache_dir)
305
306    except Exception as e:
307      logger.LogError("Exception while using cache: %s" % e)
308      return None
309    return result
310
311
312class TelemetryResult(Result):
313
314  def __init__(self, logger, label):
315    super(TelemetryResult, self).__init__(logger, label)
316
317  def _PopulateFromRun(self, out, err, retval):
318    self.out = out
319    self.err = err
320    self.retval = retval
321
322    self._ProcessResults()
323
324  def _ProcessResults(self):
325    # The output is:
326    # url,average_commit_time (ms),...
327    # www.google.com,33.4,21.2,...
328    # We need to convert to this format:
329    # {"www.google.com:average_commit_time (ms)": "33.4",
330    #  "www.google.com:...": "21.2"}
331    # Added note:  Occasionally the output comes back
332    # with "JSON.stringify(window.automation.GetResults())" on
333    # the first line, and then the rest of the output as
334    # described above.
335
336    lines = self.out.splitlines()
337    self.keyvals = {}
338
339    if lines:
340      if lines[0].startswith("JSON.stringify"):
341        lines = lines[1:]
342
343    if not lines:
344      return
345    labels = lines[0].split(",")
346    for line in lines[1:]:
347      fields = line.split(",")
348      if len(fields) != len(labels):
349        continue
350      for i in range(1, len(labels)):
351        key = "%s %s" % (fields[0], labels[i])
352        value = fields[i]
353        self.keyvals[key] = value
354    self.keyvals["retval"] = self.retval
355
356  def _PopulateFromCacheDir(self, cache_dir):
357    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
358      self.out = pickle.load(f)
359      self.err = pickle.load(f)
360      self.retval = pickle.load(f)
361    self._ProcessResults()
362
363
364class CacheConditions(object):
365  # Cache hit only if the result file exists.
366  CACHE_FILE_EXISTS = 0
367
368  # Cache hit if the checksum of cpuinfo and totalmem of
369  # the cached result and the new run match.
370  MACHINES_MATCH = 1
371
372  # Cache hit if the image checksum of the cached result and the new run match.
373  CHECKSUMS_MATCH = 2
374
375  # Cache hit only if the cached result was successful
376  RUN_SUCCEEDED = 3
377
378  # Never a cache hit.
379  FALSE = 4
380
381  # Cache hit if the image path matches the cached image path.
382  IMAGE_PATH_MATCH = 5
383
384  # Cache hit if the uuid of hard disk mataches the cached one
385
386  SAME_MACHINE_MATCH = 6
387
388
389class ResultsCache(object):
390
391  """ This class manages the key of the cached runs without worrying about what
392  is exactly stored (value). The value generation is handled by the Results
393  class.
394  """
395  CACHE_VERSION = 6
396
397  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
398           test_args, profiler_args, machine_manager, board, cache_conditions,
399           logger_to_use, label, share_users, suite):
400    self.chromeos_image = chromeos_image
401    self.chromeos_root = chromeos_root
402    self.test_name = test_name
403    self.iteration = iteration
404    self.test_args = test_args
405    self.profiler_args = profiler_args
406    self.board = board
407    self.cache_conditions = cache_conditions
408    self.machine_manager = machine_manager
409    self._logger = logger_to_use
410    self._ce = command_executer.GetCommandExecuter(self._logger)
411    self.label = label
412    self.share_users = share_users
413    self.suite = suite
414
415  def _GetCacheDirForRead(self):
416    matching_dirs = []
417    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
418      matching_dirs += glob.glob(glob_path)
419
420    if matching_dirs:
421      # Cache file found.
422      return matching_dirs[0]
423    else:
424      return None
425
426  def _GetCacheDirForWrite(self):
427    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
428
429  def _FormCacheDir(self, list_of_strings):
430    cache_key = " ".join(list_of_strings)
431    cache_dir = misc.GetFilenameFromString(cache_key)
432    if self.label.cache_dir:
433      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
434      cache_path = [os.path.join(cache_home, cache_dir)]
435    else:
436      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
437
438    for i in [x.strip() for x in self.share_users.split(",")]:
439      path = SCRATCH_BASE % i
440      cache_path.append(os.path.join(path, cache_dir))
441
442    return cache_path
443
444  def _GetCacheKeyList(self, read):
445    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
446      machine_checksum = "*"
447    else:
448      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
449    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
450      checksum = "*"
451    else:
452      checksum = ImageChecksummer().Checksum(self.label)
453
454    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
455      image_path_checksum = "*"
456    else:
457      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
458
459    machine_id_checksum = ""
460    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
461      machine_id_checksum = "*"
462    else:
463      for machine in self.machine_manager.GetMachines(self.label):
464        if machine.name == self.label.remote[0]:
465          machine_id_checksum = machine.machine_id_checksum
466          break
467
468    temp_test_args = "%s %s" % (self.test_args, self.profiler_args)
469    test_args_checksum = hashlib.md5(
470        "".join(temp_test_args)).hexdigest()
471    return (image_path_checksum,
472            self.test_name, str(self.iteration),
473            test_args_checksum,
474            checksum,
475            machine_checksum,
476            machine_id_checksum,
477            str(self.CACHE_VERSION))
478
479  def ReadResult(self):
480    if CacheConditions.FALSE in self.cache_conditions:
481      cache_dir = self._GetCacheDirForWrite()
482      command = "rm -rf {0}".format(cache_dir)
483      self._ce.RunCommand(command)
484      return None
485    cache_dir = self._GetCacheDirForRead()
486
487    if not cache_dir:
488      return None
489
490    if not os.path.isdir(cache_dir):
491      return None
492
493    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
494    result = Result.CreateFromCacheHit(self._logger,
495                                       self.label,
496                                       cache_dir,
497                                       self.suite)
498    if not result:
499      return None
500
501    if (result.retval == 0 or
502        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
503      return result
504
505    return None
506
507  def StoreResult(self, result):
508    cache_dir = self._GetCacheDirForWrite()
509    result.StoreToCacheDir(cache_dir, self.machine_manager)
510
511
512class MockResultsCache(ResultsCache):
513  def Init(self, *args):
514    pass
515
516  def ReadResult(self):
517    return None
518
519  def StoreResult(self, result):
520    pass
521
522
523class MockResult(Result):
524  def _PopulateFromRun(self, out, err, retval):
525    self.out = out
526    self.err = err
527    self.retval = retval
528