results_cache.py revision 1a224369afcbfd0276f4c7bdc625dec7f7b30d01 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to deal with result cache."""
8
9import getpass
10import glob
11import hashlib
12import os
13import pickle
14import re
15import tempfile
16import json
17import sys
18
19from utils import command_executer
20from utils import misc
21
22from image_checksummer import ImageChecksummer
23
24SCRATCH_DIR = os.path.expanduser("~/cros_scratch")
25RESULTS_FILE = "results.txt"
26MACHINE_FILE = "machine.txt"
27AUTOTEST_TARBALL = "autotest.tbz2"
28PERF_RESULTS_FILE = "perf-results.txt"
29
30class Result(object):
31  """ This class manages what exactly is stored inside the cache without knowing
32  what the key of the cache is. For runs with perf, it stores perf.data,
33  perf.report, etc. The key generation is handled by the ResultsCache class.
34  """
35
36  def __init__(self, logger, label, log_level, cmd_exec=None):
37    self._chromeos_root = label.chromeos_root
38    self._logger = logger
39    self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger,
40                                                   log_level=log_level)
41    self._temp_dir = None
42    self.label = label
43    self.results_dir = None
44    self.log_level = log_level
45    self.perf_data_files = []
46    self.perf_report_files = []
47
48  def _CopyFilesTo(self, dest_dir, files_to_copy):
49    file_index = 0
50    for file_to_copy in files_to_copy:
51      if not os.path.isdir(dest_dir):
52        command = "mkdir -p %s" % dest_dir
53        self._ce.RunCommand(command)
54      dest_file = os.path.join(dest_dir,
55                               ("%s.%s" % (os.path.basename(file_to_copy),
56                                           file_index)))
57      ret = self._ce.CopyFiles(file_to_copy,
58                               dest_file,
59                               recursive=False)
60      if ret:
61        raise Exception("Could not copy results file: %s" % file_to_copy)
62
63  def CopyResultsTo(self, dest_dir):
64    self._CopyFilesTo(dest_dir, self.perf_data_files)
65    self._CopyFilesTo(dest_dir, self.perf_report_files)
66    if len(self.perf_data_files) or len(self.perf_report_files):
67      self._logger.LogOutput("Perf results files stored in %s." % dest_dir)
68
69  def _GetNewKeyvals(self, keyvals_dict):
70    # Initialize 'units' dictionary.
71    units_dict = {}
72    for k in keyvals_dict:
73      units_dict[k] = ""
74    results_files = self._GetDataMeasurementsFiles()
75    for f in results_files:
76      # Make sure we can find the results file
77      if os.path.exists(f):
78        data_filename = f
79      else:
80        # Otherwise get the base filename and create the correct
81        # path for it.
82        f_dir, f_base = misc.GetRoot(f)
83        data_filename = os.path.join(self._chromeos_root, "chroot/tmp",
84                                     self._temp_dir, f_base)
85      if os.path.exists(data_filename):
86        with open(data_filename, "r") as data_file:
87          lines = data_file.readlines()
88          for line in lines:
89            tmp_dict = json.loads(line)
90            graph_name = tmp_dict["graph"]
91            graph_str = (graph_name + "__") if graph_name else ""
92            key = graph_str + tmp_dict["description"]
93            keyvals_dict[key] = tmp_dict["value"]
94            units_dict[key] = tmp_dict["units"]
95
96    return keyvals_dict, units_dict
97
98
99  def _AppendTelemetryUnits(self, keyvals_dict, units_dict):
100    """
101    keyvals_dict is the dictionary of key-value pairs that is used for
102    generating Crosperf reports.
103
104    units_dict is a dictionary of the units for the return values in
105    keyvals_dict.  We need to associate the units with the return values,
106    for Telemetry tests, so that we can include the units in the reports.
107    This function takes each value in keyvals_dict, finds the corresponding
108    unit in the units_dict, and replaces the old value with a list of the
109    old value and the units.  This later gets properly parsed in the
110    ResultOrganizer class, for generating the reports.
111
112    """
113
114    results_dict = {}
115    for k in keyvals_dict:
116      # We don't want these lines in our reports; they add no useful data.
117      if k == "" or k == "telemetry_Crosperf":
118        continue
119      val = keyvals_dict[k]
120      units = units_dict[k]
121      new_val = [ val, units ]
122      results_dict[k] = new_val
123    return results_dict
124
125  def _GetKeyvals(self, show_all):
126    results_in_chroot = os.path.join(self._chromeos_root,
127                                     "chroot", "tmp")
128    if not self._temp_dir:
129      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
130      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
131      self._ce.RunCommand(command)
132
133    command = ("python generate_test_report --no-color --csv %s" %
134               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
135    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
136                                            command,
137                                            return_output=True)
138    keyvals_dict = {}
139    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
140                                                 self._temp_dir)
141    for line in out.splitlines():
142      tokens = re.split("=|,", line)
143      key = tokens[-2]
144      if key.startswith(tmp_dir_in_chroot):
145        key = key[len(tmp_dir_in_chroot) + 1:]
146      value = tokens[-1]
147      keyvals_dict[key] = value
148
149    # Check to see if there is a perf_measurements file and get the
150    # data from it if so.
151    keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict)
152    if self.suite == "telemetry_Crosperf":
153      # For telemtry_Crosperf results, append the units to the return
154      # results, for use in generating the reports.
155      keyvals_dict = self._AppendTelemetryUnits(keyvals_dict,
156                                                units_dict)
157    return keyvals_dict
158
159  def _GetResultsDir(self):
160    mo = re.search(r"Results placed in (\S+)", self.out)
161    if mo:
162      result = mo.group(1)
163      return result
164    raise Exception("Could not find results directory.")
165
166  def _FindFilesInResultsDir(self, find_args):
167    if not self.results_dir:
168      return None
169
170    command = "find %s %s" % (self.results_dir,
171                              find_args)
172    ret, out, _ = self._ce.RunCommand(command, return_output=True)
173    if ret:
174      raise Exception("Could not run find command!")
175    return out
176
177  def _GetPerfDataFiles(self):
178    return self._FindFilesInResultsDir("-name perf.data").splitlines()
179
180  def _GetPerfReportFiles(self):
181    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
182
183  def _GetDataMeasurementsFiles(self):
184    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()
185
186  def _GeneratePerfReportFiles(self):
187    perf_report_files = []
188    for perf_data_file in self.perf_data_files:
189      # Generate a perf.report and store it side-by-side with the perf.data
190      # file.
191      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
192                                                       perf_data_file)
193      perf_report_file = "%s.report" % perf_data_file
194      if os.path.exists(perf_report_file):
195        raise Exception("Perf report file already exists: %s" %
196                        perf_report_file)
197      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
198                                                         perf_report_file)
199      perf_path = os.path.join (self._chromeos_root,
200                                "chroot",
201                                "usr/bin/perf")
202
203      perf_file = "/usr/sbin/perf"
204      if os.path.exists(perf_path):
205        perf_file = "/usr/bin/perf"
206
207      # The following is a hack, to use the perf.static binary that
208      # was given to us by Stephane Eranian, until he can figure out
209      # why "normal" perf cannot properly symbolize ChromeOS perf.data files.
210      # Get the directory containing the 'crosperf' script.
211      dirname, _ = misc.GetRoot(sys.argv[0])
212      perf_path = os.path.join (dirname, "..", "perf.static")
213      if os.path.exists(perf_path):
214        # copy the executable into the chroot so that it can be found.
215        src_path = perf_path
216        dst_path = os.path.join (self._chromeos_root, "chroot",
217                                 "tmp/perf.static")
218        command = "cp %s %s" % (src_path,dst_path)
219        self._ce.RunCommand (command)
220        perf_file = "/tmp/perf.static"
221
222      command = ("%s report "
223                 "-n "
224                 "--symfs /build/%s "
225                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
226                 "--kallsyms /build/%s/boot/System.map-* "
227                 "-i %s --stdio "
228                 "> %s" %
229                 (perf_file,
230                  self._board,
231                  self._board,
232                  self._board,
233                  chroot_perf_data_file,
234                  chroot_perf_report_file))
235      self._ce.ChrootRunCommand(self._chromeos_root,
236                                command)
237
238      # Add a keyval to the dictionary for the events captured.
239      perf_report_files.append(
240          misc.GetOutsideChrootPath(self._chromeos_root,
241                                    chroot_perf_report_file))
242    return perf_report_files
243
244  def _GatherPerfResults(self):
245    report_id = 0
246    for perf_report_file in self.perf_report_files:
247      with open(perf_report_file, "r") as f:
248        report_contents = f.read()
249        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
250          num_events = group[0]
251          event_name = group[1]
252          key = "perf_%s_%s" % (report_id, event_name)
253          value = str(misc.UnitToNumber(num_events))
254          self.keyvals[key] = value
255
256  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
257    self._board = self.label.board
258    self.out = out
259    self.err = err
260    self.retval = retval
261    self.test_name = test
262    self.suite = suite
263    self.chroot_results_dir = self._GetResultsDir()
264    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
265                                                 self.chroot_results_dir)
266    self.perf_data_files = self._GetPerfDataFiles()
267    # Include all perf.report data in table.
268    self.perf_report_files = self._GeneratePerfReportFiles()
269    # TODO(asharif): Do something similar with perf stat.
270
271    # Grab keyvals from the directory.
272    self._ProcessResults(show_all)
273
274  def _ProcessResults(self, show_all):
275    # Note that this function doesn't know anything about whether there is a
276    # cache hit or miss. It should process results agnostic of the cache hit
277    # state.
278    self.keyvals = self._GetKeyvals(show_all)
279    self.keyvals["retval"] = self.retval
280    # Generate report from all perf.data files.
281    # Now parse all perf report files and include them in keyvals.
282    self._GatherPerfResults()
283
284  def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite):
285    self.test_name = test
286    self.suite = suite
287    # Read in everything from the cache directory.
288    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
289      self.out = pickle.load(f)
290      self.err = pickle.load(f)
291      self.retval = pickle.load(f)
292
293    # Untar the tarball to a temporary directory
294    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
295                                                       "chroot", "tmp"))
296
297    command = ("cd %s && tar xf %s" %
298               (self._temp_dir,
299                os.path.join(cache_dir, AUTOTEST_TARBALL)))
300    ret = self._ce.RunCommand(command)
301    if ret:
302      raise Exception("Could not untar cached tarball")
303    self.results_dir = self._temp_dir
304    self.perf_data_files = self._GetPerfDataFiles()
305    self.perf_report_files = self._GetPerfReportFiles()
306    self._ProcessResults(show_all)
307
308  def CleanUp(self, rm_chroot_tmp):
309    if rm_chroot_tmp and self.results_dir:
310      dirname, basename = misc.GetRoot(self.results_dir)
311      if basename.find("test_that_results_") != -1:
312        command = "rm -rf %s" % self.results_dir
313      else:
314        command = "rm -rf %s" % dirname
315      self._ce.RunCommand(command)
316    if self._temp_dir:
317      command = "rm -rf %s" % self._temp_dir
318      self._ce.RunCommand(command)
319
320  def StoreToCacheDir(self, cache_dir, machine_manager):
321    # Create the dir if it doesn't exist.
322    temp_dir = tempfile.mkdtemp()
323
324    # Store to the temp directory.
325    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
326      pickle.dump(self.out, f)
327      pickle.dump(self.err, f)
328      pickle.dump(self.retval, f)
329
330    if self.results_dir:
331      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
332      command = ("cd %s && "
333                 "tar "
334                 "--exclude=var/spool "
335                 "--exclude=var/log "
336                 "-cjf %s ." % (self.results_dir, tarball))
337      ret = self._ce.RunCommand(command)
338      if ret:
339        raise Exception("Couldn't store autotest output directory.")
340    # Store machine info.
341    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
342    # this function.
343    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
344      f.write(machine_manager.machine_checksum_string[self.label.name])
345
346    if os.path.exists(cache_dir):
347      command = "rm -rf {0}".format(cache_dir)
348      self._ce.RunCommand(command)
349
350    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
351    command += "chmod g+x {0} && ".format(temp_dir)
352    command += "mv {0} {1}".format(temp_dir, cache_dir)
353    ret = self._ce.RunCommand(command)
354    if ret:
355      command = "rm -rf {0}".format(temp_dir)
356      self._ce.RunCommand(command)
357      raise Exception("Could not move dir %s to dir %s" %
358                      (temp_dir, cache_dir))
359
360  @classmethod
361  def CreateFromRun(cls, logger, log_level, label, out, err, retval, show_all,
362                    test, suite="telemetry_Crosperf"):
363    if suite == "telemetry":
364      result = TelemetryResult(logger, label, log_level)
365    else:
366      result = cls(logger, label, log_level)
367    result._PopulateFromRun(out, err, retval, show_all, test, suite)
368    return result
369
370  @classmethod
371  def CreateFromCacheHit(cls, logger, log_level, label, cache_dir,
372                         show_all, test, suite="telemetry_Crosperf"):
373    if suite == "telemetry":
374      result = TelemetryResult(logger, label)
375    else:
376      result = cls(logger, label, log_level)
377    try:
378      result._PopulateFromCacheDir(cache_dir, show_all, test, suite)
379
380    except Exception as e:
381      logger.LogError("Exception while using cache: %s" % e)
382      return None
383    return result
384
385
386class TelemetryResult(Result):
387
388  def __init__(self, logger, label, log_level, cmd_exec=None):
389    super(TelemetryResult, self).__init__(logger, label, log_level, cmd_exec)
390
391  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
392    self.out = out
393    self.err = err
394    self.retval = retval
395
396    self._ProcessResults()
397
398  def _ProcessResults(self):
399    # The output is:
400    # url,average_commit_time (ms),...
401    # www.google.com,33.4,21.2,...
402    # We need to convert to this format:
403    # {"www.google.com:average_commit_time (ms)": "33.4",
404    #  "www.google.com:...": "21.2"}
405    # Added note:  Occasionally the output comes back
406    # with "JSON.stringify(window.automation.GetResults())" on
407    # the first line, and then the rest of the output as
408    # described above.
409
410    lines = self.out.splitlines()
411    self.keyvals = {}
412
413    if lines:
414      if lines[0].startswith("JSON.stringify"):
415        lines = lines[1:]
416
417    if not lines:
418      return
419    labels = lines[0].split(",")
420    for line in lines[1:]:
421      fields = line.split(",")
422      if len(fields) != len(labels):
423        continue
424      for i in range(1, len(labels)):
425        key = "%s %s" % (fields[0], labels[i])
426        value = fields[i]
427        self.keyvals[key] = value
428    self.keyvals["retval"] = self.retval
429
430  def _PopulateFromCacheDir(self, cache_dir):
431    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
432      self.out = pickle.load(f)
433      self.err = pickle.load(f)
434      self.retval = pickle.load(f)
435    self._ProcessResults()
436
437
438class CacheConditions(object):
439  # Cache hit only if the result file exists.
440  CACHE_FILE_EXISTS = 0
441
442  # Cache hit if the checksum of cpuinfo and totalmem of
443  # the cached result and the new run match.
444  MACHINES_MATCH = 1
445
446  # Cache hit if the image checksum of the cached result and the new run match.
447  CHECKSUMS_MATCH = 2
448
449  # Cache hit only if the cached result was successful
450  RUN_SUCCEEDED = 3
451
452  # Never a cache hit.
453  FALSE = 4
454
455  # Cache hit if the image path matches the cached image path.
456  IMAGE_PATH_MATCH = 5
457
458  # Cache hit if the uuid of hard disk mataches the cached one
459
460  SAME_MACHINE_MATCH = 6
461
462
463class ResultsCache(object):
464
465  """ This class manages the key of the cached runs without worrying about what
466  is exactly stored (value). The value generation is handled by the Results
467  class.
468  """
469  CACHE_VERSION = 6
470
471  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
472           test_args, profiler_args, machine_manager, board, cache_conditions,
473           logger_to_use, log_level, label, share_cache, suite,
474           show_all_results):
475    self.chromeos_image = chromeos_image
476    self.chromeos_root = chromeos_root
477    self.test_name = test_name
478    self.iteration = iteration
479    self.test_args = test_args
480    self.profiler_args = profiler_args
481    self.board = board
482    self.cache_conditions = cache_conditions
483    self.machine_manager = machine_manager
484    self._logger = logger_to_use
485    self._ce = command_executer.GetCommandExecuter(self._logger,
486                                                   log_level=log_level)
487    self.label = label
488    self.share_cache = share_cache
489    self.suite = suite
490    self.log_level = log_level
491    self.show_all = show_all_results
492
493  def _GetCacheDirForRead(self):
494    matching_dirs = []
495    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
496      matching_dirs += glob.glob(glob_path)
497
498    if matching_dirs:
499      # Cache file found.
500      return matching_dirs[0]
501    else:
502      return None
503
504  def _GetCacheDirForWrite(self):
505    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
506
507  def _FormCacheDir(self, list_of_strings):
508    cache_key = " ".join(list_of_strings)
509    cache_dir = misc.GetFilenameFromString(cache_key)
510    if self.label.cache_dir:
511      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
512      cache_path = [os.path.join(cache_home, cache_dir)]
513    else:
514      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
515
516    if len(self.share_cache):
517      for path in [x.strip() for x in self.share_cache.split(",")]:
518        if os.path.exists(path):
519          cache_path.append(os.path.join(path, cache_dir))
520        else:
521          self._logger.LogFatal("Unable to find shared cache: %s" % path)
522
523    return cache_path
524
525  def _GetCacheKeyList(self, read):
526    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
527      machine_checksum = "*"
528    else:
529      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
530    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
531      checksum = "*"
532    elif self.label.image_type == "trybot":
533      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
534    elif self.label.image_type == "official":
535      checksum = "*"
536    else:
537      checksum = ImageChecksummer().Checksum(self.label, self.log_level)
538
539    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
540      image_path_checksum = "*"
541    else:
542      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
543
544    machine_id_checksum = ""
545    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
546      machine_id_checksum = "*"
547    else:
548      for machine in self.machine_manager.GetMachines(self.label):
549        if machine.name == self.label.remote[0]:
550          machine_id_checksum = machine.machine_id_checksum
551          break
552
553    temp_test_args = "%s %s" % (self.test_args, self.profiler_args)
554    test_args_checksum = hashlib.md5(
555        "".join(temp_test_args)).hexdigest()
556    return (image_path_checksum,
557            self.test_name, str(self.iteration),
558            test_args_checksum,
559            checksum,
560            machine_checksum,
561            machine_id_checksum,
562            str(self.CACHE_VERSION))
563
564  def ReadResult(self):
565    if CacheConditions.FALSE in self.cache_conditions:
566      cache_dir = self._GetCacheDirForWrite()
567      command = "rm -rf {0}".format(cache_dir)
568      self._ce.RunCommand(command)
569      return None
570    cache_dir = self._GetCacheDirForRead()
571
572    if not cache_dir:
573      return None
574
575    if not os.path.isdir(cache_dir):
576      return None
577
578    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
579    result = Result.CreateFromCacheHit(self._logger,
580                                       self.log_level,
581                                       self.label,
582                                       cache_dir,
583                                       self.show_all,
584                                       self.test_name,
585                                       self.suite)
586    if not result:
587      return None
588
589    if (result.retval == 0 or
590        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
591      return result
592
593    return None
594
595  def StoreResult(self, result):
596    cache_dir = self._GetCacheDirForWrite()
597    result.StoreToCacheDir(cache_dir, self.machine_manager)
598
599
600class MockResultsCache(ResultsCache):
601  def Init(self, *args):
602    pass
603
604  def ReadResult(self):
605    return None
606
607  def StoreResult(self, result):
608    pass
609
610
611class MockResult(Result):
612  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
613    self.out = out
614    self.err = err
615    self.retval = retval
616