results_cache.py revision 98a53692fb946a8eac46e3e82257f540d1350c18 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module to deal with result cache."""
8
9import getpass
10import glob
11import hashlib
12import os
13import pickle
14import re
15import tempfile
16import json
17import sys
18
19from utils import command_executer
20from utils import misc
21
22from image_checksummer import ImageChecksummer
23
24SCRATCH_BASE = "/home/%s/cros_scratch"
25SCRATCH_DIR = SCRATCH_BASE % getpass.getuser()
26RESULTS_FILE = "results.txt"
27MACHINE_FILE = "machine.txt"
28AUTOTEST_TARBALL = "autotest.tbz2"
29PERF_RESULTS_FILE = "perf-results.txt"
30
31class Result(object):
32  """ This class manages what exactly is stored inside the cache without knowing
33  what the key of the cache is. For runs with perf, it stores perf.data,
34  perf.report, etc. The key generation is handled by the ResultsCache class.
35  """
36
37  def __init__(self, logger, label, log_level):
38    self._chromeos_root = label.chromeos_root
39    self._logger = logger
40    self._ce = command_executer.GetCommandExecuter(self._logger,
41                                                   log_level=log_level)
42    self._temp_dir = None
43    self.label = label
44    self.results_dir = None
45    self.log_level = log_level
46    self.perf_data_files = []
47    self.perf_report_files = []
48
49  def _CopyFilesTo(self, dest_dir, files_to_copy):
50    file_index = 0
51    for file_to_copy in files_to_copy:
52      if not os.path.isdir(dest_dir):
53        command = "mkdir -p %s" % dest_dir
54        self._ce.RunCommand(command)
55      dest_file = os.path.join(dest_dir,
56                               ("%s.%s" % (os.path.basename(file_to_copy),
57                                           file_index)))
58      ret = self._ce.CopyFiles(file_to_copy,
59                               dest_file,
60                               recursive=False)
61      if ret:
62        raise Exception("Could not copy results file: %s" % file_to_copy)
63
64  def CopyResultsTo(self, dest_dir):
65    self._CopyFilesTo(dest_dir, self.perf_data_files)
66    self._CopyFilesTo(dest_dir, self.perf_report_files)
67
68  def _GetNewKeyvals(self, keyvals_dict):
69    # Initialize 'units' dictionary.
70    units_dict = {}
71    for k in keyvals_dict:
72      units_dict[k] = ""
73    results_files = self._GetDataMeasurementsFiles()
74    for f in results_files:
75      # Make sure we can find the results file
76      if os.path.exists(f):
77        data_filename = f
78      else:
79        # Otherwise get the base filename and create the correct
80        # path for it.
81        f_dir, f_base = misc.GetRoot(f)
82        data_filename = os.path.join(self._chromeos_root, "/tmp",
83                                     self._temp_dir, f_base)
84      if os.path.exists(data_filename):
85        with open(data_filename, "r") as data_file:
86          lines = data_file.readlines()
87          for line in lines:
88            tmp_dict = json.loads(line)
89            key = tmp_dict["graph"] + "__" + tmp_dict["description"]
90            keyvals_dict[key] = tmp_dict["value"]
91            units_dict[key] = tmp_dict["units"]
92
93    return keyvals_dict, units_dict
94
95
96  def _AppendTelemetryUnits(self, keyvals_dict, units_dict):
97    """
98    keyvals_dict is the dictionary of key-value pairs that is used for
99    generating Crosperf reports.
100
101    units_dict is a dictionary of the units for the return values in
102    keyvals_dict.  We need to associate the units with the return values,
103    for Telemetry tests, so that we can include the units in the reports.
104    This function takes each value in keyvals_dict, finds the corresponding
105    unit in the units_dict, and replaces the old value with a list of the
106    old value and the units.  This later gets properly parsed in the
107    ResultOrganizer class, for generating the reports.
108
109    """
110
111    results_dict = {}
112    for k in keyvals_dict:
113      # We don't want these lines in our reports; they add no useful data.
114      if k == "" or k == "telemetry_Crosperf":
115        continue
116      val = keyvals_dict[k]
117      units = units_dict[k]
118      new_val = [ val, units ]
119      results_dict[k] = new_val
120    return results_dict
121
122  def _GetKeyvals(self, show_all):
123    results_in_chroot = os.path.join(self._chromeos_root,
124                                     "chroot", "tmp")
125    if not self._temp_dir:
126      self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
127      command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir)
128      self._ce.RunCommand(command)
129
130    command = ("python generate_test_report --no-color --csv %s" %
131               (os.path.join("/tmp", os.path.basename(self._temp_dir))))
132    [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root,
133                                            command,
134                                            return_output=True)
135    keyvals_dict = {}
136    tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root,
137                                                 self._temp_dir)
138    for line in out.splitlines():
139      tokens = re.split("=|,", line)
140      key = tokens[-2]
141      if key.startswith(tmp_dir_in_chroot):
142        key = key[len(tmp_dir_in_chroot) + 1:]
143      value = tokens[-1]
144      keyvals_dict[key] = value
145
146    # Check to see if there is a perf_measurements file and get the
147    # data from it if so.
148    keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict)
149    if self.suite == "telemetry_Crosperf":
150      # For telemtry_Crosperf results, append the units to the return
151      # results, for use in generating the reports.
152      keyvals_dict = self._AppendTelemetryUnits(keyvals_dict,
153                                                units_dict)
154    return keyvals_dict
155
156  def _GetResultsDir(self):
157    mo = re.search(r"Results placed in (\S+)", self.out)
158    if mo:
159      result = mo.group(1)
160      return result
161    raise Exception("Could not find results directory.")
162
163  def _FindFilesInResultsDir(self, find_args):
164    if not self.results_dir:
165      return None
166
167    command = "find %s %s" % (self.results_dir,
168                              find_args)
169    ret, out, _ = self._ce.RunCommand(command, return_output=True)
170    if ret:
171      raise Exception("Could not run find command!")
172    return out
173
174  def _GetPerfDataFiles(self):
175    return self._FindFilesInResultsDir("-name perf.data").splitlines()
176
177  def _GetPerfReportFiles(self):
178    return self._FindFilesInResultsDir("-name perf.data.report").splitlines()
179
180  def _GetDataMeasurementsFiles(self):
181    return self._FindFilesInResultsDir("-name perf_measurements").splitlines()
182
183  def _GeneratePerfReportFiles(self):
184    perf_report_files = []
185    for perf_data_file in self.perf_data_files:
186      # Generate a perf.report and store it side-by-side with the perf.data
187      # file.
188      chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root,
189                                                       perf_data_file)
190      perf_report_file = "%s.report" % perf_data_file
191      if os.path.exists(perf_report_file):
192        raise Exception("Perf report file already exists: %s" %
193                        perf_report_file)
194      chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root,
195                                                         perf_report_file)
196      perf_path = os.path.join (self._chromeos_root,
197                                "chroot",
198                                "usr/bin/perf")
199
200      perf_file = "/usr/sbin/perf"
201      if os.path.exists(perf_path):
202        perf_file = "/usr/bin/perf"
203
204      # The following is a hack, to use the perf.static binary that
205      # was given to us by Stephane Eranian, until he can figure out
206      # why "normal" perf cannot properly symbolize ChromeOS perf.data files.
207      # Get the directory containing the 'crosperf' script.
208      dirname, _ = misc.GetRoot(sys.argv[0])
209      perf_path = os.path.join (dirname, "..", "perf.static")
210      if os.path.exists(perf_path):
211        # copy the executable into the chroot so that it can be found.
212        src_path = perf_path
213        dst_path = os.path.join (self._chromeos_root, "chroot",
214                                 "tmp/perf.static")
215        command = "cp %s %s" % (src_path,dst_path)
216        self._ce.RunCommand (command)
217        perf_file = "/tmp/perf.static"
218
219      command = ("%s report "
220                 "-n "
221                 "--symfs /build/%s "
222                 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux "
223                 "--kallsyms /build/%s/boot/System.map-* "
224                 "-i %s --stdio "
225                 "> %s" %
226                 (perf_file,
227                  self._board,
228                  self._board,
229                  self._board,
230                  chroot_perf_data_file,
231                  chroot_perf_report_file))
232      self._ce.ChrootRunCommand(self._chromeos_root,
233                                command)
234
235      # Add a keyval to the dictionary for the events captured.
236      perf_report_files.append(
237          misc.GetOutsideChrootPath(self._chromeos_root,
238                                    chroot_perf_report_file))
239    return perf_report_files
240
241  def _GatherPerfResults(self):
242    report_id = 0
243    for perf_report_file in self.perf_report_files:
244      with open(perf_report_file, "r") as f:
245        report_contents = f.read()
246        for group in re.findall(r"Events: (\S+) (\S+)", report_contents):
247          num_events = group[0]
248          event_name = group[1]
249          key = "perf_%s_%s" % (report_id, event_name)
250          value = str(misc.UnitToNumber(num_events))
251          self.keyvals[key] = value
252
253  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
254    self._board = self.label.board
255    self.out = out
256    self.err = err
257    self.retval = retval
258    self.test_name = test
259    self.suite = suite
260    self.chroot_results_dir = self._GetResultsDir()
261    self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root,
262                                                 self.chroot_results_dir)
263    self.perf_data_files = self._GetPerfDataFiles()
264    # Include all perf.report data in table.
265    self.perf_report_files = self._GeneratePerfReportFiles()
266    # TODO(asharif): Do something similar with perf stat.
267
268    # Grab keyvals from the directory.
269    self._ProcessResults(show_all)
270
271  def _ProcessResults(self, show_all):
272    # Note that this function doesn't know anything about whether there is a
273    # cache hit or miss. It should process results agnostic of the cache hit
274    # state.
275    self.keyvals = self._GetKeyvals(show_all)
276    self.keyvals["retval"] = self.retval
277    # Generate report from all perf.data files.
278    # Now parse all perf report files and include them in keyvals.
279    self._GatherPerfResults()
280
281  def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite):
282    self.test_name = test
283    self.suite = suite
284    # Read in everything from the cache directory.
285    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
286      self.out = pickle.load(f)
287      self.err = pickle.load(f)
288      self.retval = pickle.load(f)
289
290    # Untar the tarball to a temporary directory
291    self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root,
292                                                       "chroot", "tmp"))
293
294    command = ("cd %s && tar xf %s" %
295               (self._temp_dir,
296                os.path.join(cache_dir, AUTOTEST_TARBALL)))
297    ret = self._ce.RunCommand(command)
298    if ret:
299      raise Exception("Could not untar cached tarball")
300    self.results_dir = self._temp_dir
301    self.perf_data_files = self._GetPerfDataFiles()
302    self.perf_report_files = self._GetPerfReportFiles()
303    self._ProcessResults(show_all)
304
305  def CleanUp(self, rm_chroot_tmp):
306    if rm_chroot_tmp and self.results_dir:
307      dirname, basename = misc.GetRoot(self.results_dir)
308      if basename.find("test_that_results_") != -1:
309        command = "rm -rf %s" % self.results_dir
310      else:
311        command = "rm -rf %s" % dirname
312      self._ce.RunCommand(command)
313    if self._temp_dir:
314      command = "rm -rf %s" % self._temp_dir
315      self._ce.RunCommand(command)
316
317  def StoreToCacheDir(self, cache_dir, machine_manager):
318    # Create the dir if it doesn't exist.
319    temp_dir = tempfile.mkdtemp()
320
321    # Store to the temp directory.
322    with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f:
323      pickle.dump(self.out, f)
324      pickle.dump(self.err, f)
325      pickle.dump(self.retval, f)
326
327    if self.results_dir:
328      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
329      command = ("cd %s && "
330                 "tar "
331                 "--exclude=var/spool "
332                 "--exclude=var/log "
333                 "-cjf %s ." % (self.results_dir, tarball))
334      ret = self._ce.RunCommand(command)
335      if ret:
336        raise Exception("Couldn't store autotest output directory.")
337    # Store machine info.
338    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
339    # this function.
340    with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
341      f.write(machine_manager.machine_checksum_string[self.label.name])
342
343    if os.path.exists(cache_dir):
344      command = "rm -rf {0}".format(cache_dir)
345      self._ce.RunCommand(command)
346
347    command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir))
348    command += "chmod g+x {0} && ".format(temp_dir)
349    command += "mv {0} {1}".format(temp_dir, cache_dir)
350    ret = self._ce.RunCommand(command)
351    if ret:
352      command = "rm -rf {0}".format(temp_dir)
353      self._ce.RunCommand(command)
354      raise Exception("Could not move dir %s to dir %s" %
355                      (temp_dir, cache_dir))
356
357  @classmethod
358  def CreateFromRun(cls, logger, log_level, label, out, err, retval, show_all,
359                    test, suite="telemetry_Crosperf"):
360    if suite == "telemetry":
361      result = TelemetryResult(logger, label, log_level)
362    else:
363      result = cls(logger, label, log_level)
364    result._PopulateFromRun(out, err, retval, show_all, test, suite)
365    return result
366
367  @classmethod
368  def CreateFromCacheHit(cls, logger, log_level, label, cache_dir,
369                         show_all, test, suite="telemetry_Crosperf"):
370    if suite == "telemetry":
371      result = TelemetryResult(logger, label)
372    else:
373      result = cls(logger, label, log_level)
374    try:
375      result._PopulateFromCacheDir(cache_dir, show_all, test, suite)
376
377    except Exception as e:
378      logger.LogError("Exception while using cache: %s" % e)
379      return None
380    return result
381
382
383class TelemetryResult(Result):
384
385  def __init__(self, logger, label, log_level):
386    super(TelemetryResult, self).__init__(logger, label, log_level)
387
388  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
389    self.out = out
390    self.err = err
391    self.retval = retval
392
393    self._ProcessResults()
394
395  def _ProcessResults(self):
396    # The output is:
397    # url,average_commit_time (ms),...
398    # www.google.com,33.4,21.2,...
399    # We need to convert to this format:
400    # {"www.google.com:average_commit_time (ms)": "33.4",
401    #  "www.google.com:...": "21.2"}
402    # Added note:  Occasionally the output comes back
403    # with "JSON.stringify(window.automation.GetResults())" on
404    # the first line, and then the rest of the output as
405    # described above.
406
407    lines = self.out.splitlines()
408    self.keyvals = {}
409
410    if lines:
411      if lines[0].startswith("JSON.stringify"):
412        lines = lines[1:]
413
414    if not lines:
415      return
416    labels = lines[0].split(",")
417    for line in lines[1:]:
418      fields = line.split(",")
419      if len(fields) != len(labels):
420        continue
421      for i in range(1, len(labels)):
422        key = "%s %s" % (fields[0], labels[i])
423        value = fields[i]
424        self.keyvals[key] = value
425    self.keyvals["retval"] = self.retval
426
427  def _PopulateFromCacheDir(self, cache_dir):
428    with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f:
429      self.out = pickle.load(f)
430      self.err = pickle.load(f)
431      self.retval = pickle.load(f)
432    self._ProcessResults()
433
434
435class CacheConditions(object):
436  # Cache hit only if the result file exists.
437  CACHE_FILE_EXISTS = 0
438
439  # Cache hit if the checksum of cpuinfo and totalmem of
440  # the cached result and the new run match.
441  MACHINES_MATCH = 1
442
443  # Cache hit if the image checksum of the cached result and the new run match.
444  CHECKSUMS_MATCH = 2
445
446  # Cache hit only if the cached result was successful
447  RUN_SUCCEEDED = 3
448
449  # Never a cache hit.
450  FALSE = 4
451
452  # Cache hit if the image path matches the cached image path.
453  IMAGE_PATH_MATCH = 5
454
455  # Cache hit if the uuid of hard disk mataches the cached one
456
457  SAME_MACHINE_MATCH = 6
458
459
460class ResultsCache(object):
461
462  """ This class manages the key of the cached runs without worrying about what
463  is exactly stored (value). The value generation is handled by the Results
464  class.
465  """
466  CACHE_VERSION = 6
467
468  def Init(self, chromeos_image, chromeos_root, test_name, iteration,
469           test_args, profiler_args, machine_manager, board, cache_conditions,
470           logger_to_use, log_level, label, share_users, suite,
471           show_all_results):
472    self.chromeos_image = chromeos_image
473    self.chromeos_root = chromeos_root
474    self.test_name = test_name
475    self.iteration = iteration
476    self.test_args = test_args
477    self.profiler_args = profiler_args
478    self.board = board
479    self.cache_conditions = cache_conditions
480    self.machine_manager = machine_manager
481    self._logger = logger_to_use
482    self._ce = command_executer.GetCommandExecuter(self._logger,
483                                                   log_level=log_level)
484    self.label = label
485    self.share_users = share_users
486    self.suite = suite
487    self.log_level = log_level
488    self.show_all = show_all_results
489
490  def _GetCacheDirForRead(self):
491    matching_dirs = []
492    for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)):
493      matching_dirs += glob.glob(glob_path)
494
495    if matching_dirs:
496      # Cache file found.
497      return matching_dirs[0]
498    else:
499      return None
500
501  def _GetCacheDirForWrite(self):
502    return self._FormCacheDir(self._GetCacheKeyList(False))[0]
503
504  def _FormCacheDir(self, list_of_strings):
505    cache_key = " ".join(list_of_strings)
506    cache_dir = misc.GetFilenameFromString(cache_key)
507    if self.label.cache_dir:
508      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
509      cache_path = [os.path.join(cache_home, cache_dir)]
510    else:
511      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
512
513    for i in [x.strip() for x in self.share_users.split(",")]:
514      path = SCRATCH_BASE % i
515      cache_path.append(os.path.join(path, cache_dir))
516
517    return cache_path
518
519  def _GetCacheKeyList(self, read):
520    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
521      machine_checksum = "*"
522    else:
523      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
524    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
525      checksum = "*"
526    elif self.label.image_type == "trybot":
527      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
528    elif self.label.image_type == "official":
529      checksum = "*"
530    else:
531      checksum = ImageChecksummer().Checksum(self.label, self.log_level)
532
533    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
534      image_path_checksum = "*"
535    else:
536      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
537
538    machine_id_checksum = ""
539    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
540      machine_id_checksum = "*"
541    else:
542      for machine in self.machine_manager.GetMachines(self.label):
543        if machine.name == self.label.remote[0]:
544          machine_id_checksum = machine.machine_id_checksum
545          break
546
547    temp_test_args = "%s %s" % (self.test_args, self.profiler_args)
548    test_args_checksum = hashlib.md5(
549        "".join(temp_test_args)).hexdigest()
550    return (image_path_checksum,
551            self.test_name, str(self.iteration),
552            test_args_checksum,
553            checksum,
554            machine_checksum,
555            machine_id_checksum,
556            str(self.CACHE_VERSION))
557
558  def ReadResult(self):
559    if CacheConditions.FALSE in self.cache_conditions:
560      cache_dir = self._GetCacheDirForWrite()
561      command = "rm -rf {0}".format(cache_dir)
562      self._ce.RunCommand(command)
563      return None
564    cache_dir = self._GetCacheDirForRead()
565
566    if not cache_dir:
567      return None
568
569    if not os.path.isdir(cache_dir):
570      return None
571
572    self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
573    result = Result.CreateFromCacheHit(self._logger,
574                                       self.log_level,
575                                       self.label,
576                                       cache_dir,
577                                       self.show_all,
578                                       self.test_name,
579                                       self.suite)
580    if not result:
581      return None
582
583    if (result.retval == 0 or
584        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
585      return result
586
587    return None
588
589  def StoreResult(self, result):
590    cache_dir = self._GetCacheDirForWrite()
591    result.StoreToCacheDir(cache_dir, self.machine_manager)
592
593
594class MockResultsCache(ResultsCache):
595  def Init(self, *args):
596    pass
597
598  def ReadResult(self):
599    return None
600
601  def StoreResult(self, result):
602    pass
603
604
605class MockResult(Result):
606  def _PopulateFromRun(self, out, err, retval, show_all, test, suite):
607    self.out = out
608    self.err = err
609    self.retval = retval
610