results_cache.py revision 75a28ac9bd52db76d106abc8e23678b0192b3a69 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to deal with result cache."""
8
9import getpass
10import glob
11import hashlib
12import os
13import pickle
14import re
15import tempfile
16import json
17import sys
18
19from utils import command_executer
20from utils import misc
21
22from image_checksummer import ImageChecksummer
23
24SCRATCH_BASE = "/home/%s/cros_scratch"
25SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
26RESULTS_FILE = "results.txt"
27MACHINE_FILE = "machine.txt"
28AUTOTEST_TARBALL = "autotest.tbz2"
29PERF_RESULTS_FILE = "perf-results.txt"
30TELEMETRY_RESULT_DEFAULTS_FILE = "default-telemetry-results.json"
31
32class Result(object):
33  """ This class manages what exactly is stored inside the cache without knowing
34  what the key of the cache is. For runs with perf, it stores perf.data,
35  perf.report, etc. The key generation is handled by the ResultsCache class.
36  """
37
38  def __init__(self, logger, label):
39    self._chromeos_root = label.chromeos_root
40    self._logger = logger
41    self._ce = command_executer.GetCommandExecuter(self._logger)
42    self._temp_dir = None
43    self.label = label
44    self.results_dir = None
45    self.perf_data_files = []
46    self.perf_report_files = []
47
48  def _CopyFilesTo(self, dest_dir, files_to_copy):
49    file_index = 0
50    for file_to_copy in files_to_copy:
51      if not os.path.isdir(dest_dir):
52        command = "mkdir -p %s" % dest_dir
53        self._ce.RunCommand(command)
54      dest_file = os.path.join(dest_dir,
55                               ("%s.%s" % (os.path.basename(file_to_copy),
56                                           file_index)))
57      ret = self._ce.CopyFiles(file_to_copy,
58                               dest_file,
59                               recursive=False)
60      if ret:
61        raise Exception("Could not copy results file: %s" % file_to_copy)
62
63  def CopyResultsTo(self, dest_dir):
64    self._CopyFilesTo(dest_dir, self.perf_data_files)
65    self._CopyFilesTo(dest_dir, self.perf_report_files)
66
67  def _GetNewKeyvals(self, keyvals_dict):
68    # Initialize 'units' dictionary.
69    units_dict = {}
70    for k in keyvals_dict:
71      units_dict[k] = ""
72    results_files = self._GetDataMeasurementsFiles()
73    for f in results_files:
74      # Make sure we can find the results file
75      if os.path.exists(f):
76        data_filename = f
77      else:
78        # Otherwise get the base filename and create the correct
79        # path for it.
80        f_dir, f_base = misc.GetRoot(f)
81        data_filename = os.path.join(self._chromeos_root, "/tmp",
82                                     self._temp_dir, f_base)
83      if os.path.exists(data_filename):
84        with open(data_filename, "r") as data_file:
85          lines = data_file.readlines()
86          for line in lines:
87            tmp_dict = json.loads(line)
88            key = tmp_dict["graph"] + "__" + tmp_dict["description"]
89            keyvals_dict[key] = tmp_dict["value"]
90            units_dict[key] = tmp_dict["units"]
91
92    return keyvals_dict, units_dict
93
94
95  def _GetTelemetryResultsKeyvals(self, keyvals_dict, units_dict):
96    """
97    keyvals_dict is the dictionary of key-value pairs that is used for
98    generating Crosperf reports.
99
100    Telemetry tests return many values (fields) that are not of
101    interest, so we have created a json file that indicates, for each
102    Telemetry benchmark, what the default return fields of interest
103    are.
104
105    units_dict is a dictionary of the units for the return values in
106    keyvals_dict.  After looking for the keys in the keyvals_dict in
107    the json file of "interesting" default return fields, we append
108    the units to the name of the field, to make the report easier to
109    understand.  We don't append the units to the results name earlier,
110    because the units are not part of the field names in the json file.
111
112    This function reads that file into a dictionary, and finds the
113    entry for the current benchmark (if it exists).  The entry
114    contains a list of return fields to use in the report.  For each
115    field in the default list, we look for the field in the input
116    keyvals_dict, and if we find it we copy the entry into our results
117    dictionary. We then return the results dictionary, which gets used
118    for actually generating the report.
119    """
120
121
122    # Check to see if telemetry_Crosperf succeeded; if not, there's no point
123    # in going further...
124
125    succeeded = False
126    if "telemetry_Crosperf" in keyvals_dict:
127      if keyvals_dict["telemetry_Crosperf"] == "PASS":
128        succeeded = True
129
130    if not succeeded:
131      return keyvals_dict
132
133    # Find the Crosperf directory, and look there for the telemetry
134    # results defaults file, if it exists.
135    results_dict = {}
136    dirname, basename = misc.GetRoot(sys.argv[0])
137    fullname = os.path.join(dirname, TELEMETRY_RESULT_DEFAULTS_FILE)
138    if os.path.exists (fullname):
139      # Slurp the file into a dictionary.  The keys in the dictionary are
140      # the benchmark names.  The value for a key is a list containing the
141      # names of all the result fields that should be returned in a 'default'
142      # report.
143      result_defaults = json.load(open(fullname))
144      # Check to see if the current benchmark test actually has an entry in
145      # the dictionary.
146      if self.test_name and self.test_name in result_defaults:
147        result_list = result_defaults[self.test_name]
148        # We have the default results list.  Make sure it's not empty...
149        if len(result_list) > 0:
150          # ...look for each default result in the dictionary of actual
151          # result fields returned. If found, add the field and its value
152          # to our final results dictionary.
153          for r in result_list:
154            if r in keyvals_dict:
155              val = keyvals_dict[r]
156              units = units_dict[r]
157              # Add the units to the key name, for the report.
158              newkey = r + " (" + units + ")"
159              results_dict[newkey] = val
160    if len(results_dict) == 0:
161      # We did not find/create any new entries.  Therefore use the keyvals_dict
162      # that was passed in, but update the entry names to have the units.
163      for k in keyvals_dict:
164        val = keyvals_dict[k]
165        units = units_dict[k]
166        newkey = k + " (" + units + ")"
167        results_dict[newkey] = val
168    keyvals_dict = results_dict
169    return keyvals_dict
170
171  def _GetKeyvals(self, show_all):
172    results_in_chroot = os.path.join(self._chromeos_root,
173                                     "chroot", "tmp")
174    if not self._temp_dir:
175      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
176      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
177      self._ce.RunCommand(command)
178
179    command = ("python generate_test_report --no-color --csv %s" %
180               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
181    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
182                                            command,
183                                            return_output=True)
184    keyvals_dict = {}
185    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
186                                                 self._temp_dir)
187    for line in out.splitlines():
188      tokens = re.split("=|,", line)
189      key = tokens[-2]
190      if key.startswith(tmp_dir_in_chroot):
191        key = key[len(tmp_dir_in_chroot) + 1:]
192      value = tokens[-1]
193      keyvals_dict[key] = value
194
195    # Check to see if there is a perf_measurements file and get the
196    # data from it if so.
197    keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict)
198    if not show_all and self.suite == "telemetry_Crosperf":
199      # We're running telemetry tests and the user did not ask to
200      # see all the results, so get the default results, to be used
201      # for generating the report.
202      keyvals_dict = self._GetTelemetryResultsKeyvals(keyvals_dict,
203                                                      units_dict)
204    return keyvals_dict
205
206  def _GetResultsDir(self):
207    mo = re.search(r"Results placed in (\S+)", self.out)
208    if mo:
209      result = mo.group(1)
210      return result
211    raise Exception("Could not find results directory.")
212
213  def _FindFilesInResultsDir(self, find_args):
214    if not self.results_dir:
215      return None
216
217    command = "find %s %s" % (self.results_dir,
218                              find_args)
219    ret, out, _ = self._ce.RunCommand(command, return_output=True)
220    if ret:
221      raise Exception("Could not run find command!")
222    return out
223
224  def _GetPerfDataFiles(self):
225    return self._FindFilesInResultsDir("-name perf.data").splitlines()
226
227  def _GetPerfReportFiles(self):
228    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
229
230  def _GetDataMeasurementsFiles(self):
231    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()
232
233  def _GeneratePerfReportFiles(self):
234    perf_report_files = []
235    for perf_data_file in self.perf_data_files:
236      # Generate a perf.report and store it side-by-side with the perf.data
237      # file.
238      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
239                                                       perf_data_file)
240      perf_report_file = "%s.report" % perf_data_file
241      if os.path.exists(perf_report_file):
242        raise Exception("Perf report file already exists: %s" %
243                        perf_report_file)
244      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
245                                                         perf_report_file)
246      command = ("/usr/sbin/perf report "
247                 "-n "
248                 "--symfs /build/%s "
249                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
250                 "--kallsyms /build/%s/boot/System.map-* "
251                 "-i %s --stdio "
252                 "> %s" %
253                 (self._board,
254                  self._board,
255                  self._board,
256                  chroot_perf_data_file,
257                  chroot_perf_report_file))
258      self._ce.ChrootRunCommand(self._chromeos_root,
259                                command)
260
261      # Add a keyval to the dictionary for the events captured.
262      perf_report_files.append(
263          misc.GetOutsideChrootPath(self._chromeos_root,
264                                    chroot_perf_report_file))
265    return perf_report_files
266
267  def _GatherPerfResults(self):
268    report_id = 0
269    for perf_report_file in self.perf_report_files:
270      with open(perf_report_file, "r") as f:
271        report_contents = f.read()
272        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
273          num_events = group[0]
274          event_name = group[1]
275          key = "perf_%s_%s" % (report_id, event_name)
276          value = str(misc.UnitToNumber(num_events))
277          self.keyvals[key] = value
278
279  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
280    self._board = self.label.board
281    self.out = out
282    self.err = err
283    self.retval = retval
284    self.test_name = test
285    self.suite = suite
286    self.chroot_results_dir = self._GetResultsDir()
287    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
288                                                 self.chroot_results_dir)
289    self.perf_data_files = self._GetPerfDataFiles()
290    # Include all perf.report data in table.
291    self.perf_report_files = self._GeneratePerfReportFiles()
292    # TODO(asharif): Do something similar with perf stat.
293
294    # Grab keyvals from the directory.
295    self._ProcessResults(show_all)
296
297  def _ProcessResults(self, show_all):
298    # Note that this function doesn't know anything about whether there is a
299    # cache hit or miss. It should process results agnostic of the cache hit
300    # state.
301    self.keyvals = self._GetKeyvals(show_all)
302    self.keyvals["retval"] = self.retval
303    # Generate report from all perf.data files.
304    # Now parse all perf report files and include them in keyvals.
305    self._GatherPerfResults()
306
307  def _PopulateFromCacheDir(self, cache_dir):
308    # Read in everything from the cache directory.
309    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
310      self.out = pickle.load(f)
311      self.err = pickle.load(f)
312      self.retval = pickle.load(f)
313
314    # Untar the tarball to a temporary directory
315    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
316                                                       "chroot", "tmp"))
317
318    command = ("cd %s && tar xf %s" %
319               (self._temp_dir,
320                os.path.join(cache_dir, AUTOTEST_TARBALL)))
321    ret = self._ce.RunCommand(command)
322    if ret:
323      raise Exception("Could not untar cached tarball")
324    self.results_dir = self._temp_dir
325    self.perf_data_files = self._GetPerfDataFiles()
326    self.perf_report_files = self._GetPerfReportFiles()
327    self._ProcessResults()
328
329  def CleanUp(self, rm_chroot_tmp):
330    if rm_chroot_tmp and self.results_dir:
331      command = "rm -rf %s" % self.results_dir
332      self._ce.RunCommand(command)
333    if self._temp_dir:
334      command = "rm -rf %s" % self._temp_dir
335      self._ce.RunCommand(command)
336
337  def StoreToCacheDir(self, cache_dir, machine_manager):
338    # Create the dir if it doesn't exist.
339    temp_dir = tempfile.mkdtemp()
340
341    # Store to the temp directory.
342    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
343      pickle.dump(self.out, f)
344      pickle.dump(self.err, f)
345      pickle.dump(self.retval, f)
346
347    if self.results_dir:
348      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
349      command = ("cd %s && "
350                 "tar "
351                 "--exclude=var/spool "
352                 "--exclude=var/log "
353                 "-cjf %s ." % (self.results_dir, tarball))
354      ret = self._ce.RunCommand(command)
355      if ret:
356        raise Exception("Couldn't store autotest output directory.")
357    # Store machine info.
358    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
359    # this function.
360    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
361      f.write(machine_manager.machine_checksum_string[self.label.name])
362
363    if os.path.exists(cache_dir):
364      command = "rm -rf {0}".format(cache_dir)
365      self._ce.RunCommand(command)
366
367    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
368    command += "chmod g+x {0} && ".format(temp_dir)
369    command += "mv {0} {1}".format(temp_dir, cache_dir)
370    ret = self._ce.RunCommand(command)
371    if ret:
372      command = "rm -rf {0}".format(temp_dir)
373      self._ce.RunCommand(command)
374      raise Exception("Could not move dir %s to dir %s" %
375                      (temp_dir, cache_dir))
376
377  @classmethod
378  def CreateFromRun(cls, logger, label, out, err, retval, show_all, test,
379                    suite="pyauto"):
380    if suite == "telemetry":
381      result = TelemetryResult(logger, label)
382    else:
383      result = cls(logger, label)
384    result._PopulateFromRun(out, err, retval, show_all, test, suite)
385    return result
386
387  @classmethod
388  def CreateFromCacheHit(cls, logger, label, cache_dir,
389                         suite="pyauto"):
390    if suite == "telemetry":
391      result = TelemetryResult(logger, label)
392    else:
393      result = cls(logger, label)
394    try:
395      result._PopulateFromCacheDir(cache_dir)
396
397    except Exception as e:
398      logger.LogError("Exception while using cache: %s" % e)
399      return None
400    return result
401
402
403class TelemetryResult(Result):
404
405  def __init__(self, logger, label):
406    super(TelemetryResult, self).__init__(logger, label)
407
408  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
409    self.out = out
410    self.err = err
411    self.retval = retval
412
413    self._ProcessResults()
414
415  def _ProcessResults(self):
416    # The output is:
417    # url,average_commit_time (ms),...
418    # www.google.com,33.4,21.2,...
419    # We need to convert to this format:
420    # {"www.google.com:average_commit_time (ms)": "33.4",
421    #  "www.google.com:...": "21.2"}
422    # Added note:  Occasionally the output comes back
423    # with "JSON.stringify(window.automation.GetResults())" on
424    # the first line, and then the rest of the output as
425    # described above.
426
427    lines = self.out.splitlines()
428    self.keyvals = {}
429
430    if lines:
431      if lines[0].startswith("JSON.stringify"):
432        lines = lines[1:]
433
434    if not lines:
435      return
436    labels = lines[0].split(",")
437    for line in lines[1:]:
438      fields = line.split(",")
439      if len(fields) != len(labels):
440        continue
441      for i in range(1, len(labels)):
442        key = "%s %s" % (fields[0], labels[i])
443        value = fields[i]
444        self.keyvals[key] = value
445    self.keyvals["retval"] = self.retval
446
447  def _PopulateFromCacheDir(self, cache_dir):
448    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
449      self.out = pickle.load(f)
450      self.err = pickle.load(f)
451      self.retval = pickle.load(f)
452    self._ProcessResults()
453
454
455class CacheConditions(object):
456  # Cache hit only if the result file exists.
457  CACHE_FILE_EXISTS = 0
458
459  # Cache hit if the checksum of cpuinfo and totalmem of
460  # the cached result and the new run match.
461  MACHINES_MATCH = 1
462
463  # Cache hit if the image checksum of the cached result and the new run match.
464  CHECKSUMS_MATCH = 2
465
466  # Cache hit only if the cached result was successful
467  RUN_SUCCEEDED = 3
468
469  # Never a cache hit.
470  FALSE = 4
471
472  # Cache hit if the image path matches the cached image path.
473  IMAGE_PATH_MATCH = 5
474
475  # Cache hit if the uuid of hard disk mataches the cached one
476
477  SAME_MACHINE_MATCH = 6
478
479
480class ResultsCache(object):
481
482  """ This class manages the key of the cached runs without worrying about what
483  is exactly stored (value). The value generation is handled by the Results
484  class.
485  """
486  CACHE_VERSION = 6
487
488  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
489           test_args, profiler_args, machine_manager, board, cache_conditions,
490           logger_to_use, label, share_users, suite):
491    self.chromeos_image = chromeos_image
492    self.chromeos_root = chromeos_root
493    self.test_name = test_name
494    self.iteration = iteration
495    self.test_args = test_args
496    self.profiler_args = profiler_args
497    self.board = board
498    self.cache_conditions = cache_conditions
499    self.machine_manager = machine_manager
500    self._logger = logger_to_use
501    self._ce = command_executer.GetCommandExecuter(self._logger)
502    self.label = label
503    self.share_users = share_users
504    self.suite = suite
505
506  def _GetCacheDirForRead(self):
507    matching_dirs = []
508    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
509      matching_dirs += glob.glob(glob_path)
510
511    if matching_dirs:
512      # Cache file found.
513      return matching_dirs[0]
514    else:
515      return None
516
517  def _GetCacheDirForWrite(self):
518    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
519
520  def _FormCacheDir(self, list_of_strings):
521    cache_key = " ".join(list_of_strings)
522    cache_dir = misc.GetFilenameFromString(cache_key)
523    if self.label.cache_dir:
524      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
525      cache_path = [os.path.join(cache_home, cache_dir)]
526    else:
527      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
528
529    for i in [x.strip() for x in self.share_users.split(",")]:
530      path = SCRATCH_BASE % i
531      cache_path.append(os.path.join(path, cache_dir))
532
533    return cache_path
534
535  def _GetCacheKeyList(self, read):
536    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
537      machine_checksum = "*"
538    else:
539      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
540    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
541      checksum = "*"
542    else:
543      checksum = ImageChecksummer().Checksum(self.label)
544
545    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
546      image_path_checksum = "*"
547    else:
548      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
549
550    machine_id_checksum = ""
551    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
552      machine_id_checksum = "*"
553    else:
554      for machine in self.machine_manager.GetMachines(self.label):
555        if machine.name == self.label.remote[0]:
556          machine_id_checksum = machine.machine_id_checksum
557          break
558
559    temp_test_args = "%s %s" % (self.test_args, self.profiler_args)
560    test_args_checksum = hashlib.md5(
561        "".join(temp_test_args)).hexdigest()
562    return (image_path_checksum,
563            self.test_name, str(self.iteration),
564            test_args_checksum,
565            checksum,
566            machine_checksum,
567            machine_id_checksum,
568            str(self.CACHE_VERSION))
569
570  def ReadResult(self):
571    if CacheConditions.FALSE in self.cache_conditions:
572      cache_dir = self._GetCacheDirForWrite()
573      command = "rm -rf {0}".format(cache_dir)
574      self._ce.RunCommand(command)
575      return None
576    cache_dir = self._GetCacheDirForRead()
577
578    if not cache_dir:
579      return None
580
581    if not os.path.isdir(cache_dir):
582      return None
583
584    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
585    result = Result.CreateFromCacheHit(self._logger,
586                                       self.label,
587                                       cache_dir,
588                                       self.suite)
589    if not result:
590      return None
591
592    if (result.retval == 0 or
593        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
594      return result
595
596    return None
597
598  def StoreResult(self, result):
599    cache_dir = self._GetCacheDirForWrite()
600    result.StoreToCacheDir(cache_dir, self.machine_manager)
601
602
603class MockResultsCache(ResultsCache):
604  def Init(self, *args):
605    pass
606
607  def ReadResult(self):
608    return None
609
610  def StoreResult(self, result):
611    pass
612
613
614class MockResult(Result):
615  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
616    self.out = out
617    self.err = err
618    self.retval = retval
619