results_cache.py revision 655dfa3bd83f6abbc7fe7cc0fa14ad38f379dcb6
1
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Module to deal with result cache."""
6
7import getpass
8import glob
9import hashlib
10import os
11import pickle
12import re
13import tempfile
14import json
15import sys
16
17from cros_utils import command_executer
18from cros_utils import misc
19
20from image_checksummer import ImageChecksummer
21
22import results_report
23import test_flag
24
25SCRATCH_DIR = os.path.expanduser('~/cros_scratch')
26RESULTS_FILE = 'results.txt'
27MACHINE_FILE = 'machine.txt'
28AUTOTEST_TARBALL = 'autotest.tbz2'
29PERF_RESULTS_FILE = 'perf-results.txt'
30CACHE_KEYS_FILE = 'cache_keys.txt'
31
32
33class Result(object):
34  """ This class manages what exactly is stored inside the cache without knowing
35  what the key of the cache is. For runs with perf, it stores perf.data,
36  perf.report, etc. The key generation is handled by the ResultsCache class.
37  """
38
39  def __init__(self, logger, label, log_level, machine, cmd_exec=None):
40    self.chromeos_root = label.chromeos_root
41    self._logger = logger
42    self.ce = cmd_exec or command_executer.GetCommandExecuter(
43        self._logger,
44        log_level=log_level)
45    self.temp_dir = None
46    self.label = label
47    self.results_dir = None
48    self.log_level = log_level
49    self.machine = machine
50    self.perf_data_files = []
51    self.perf_report_files = []
52
53  def CopyFilesTo(self, dest_dir, files_to_copy):
54    file_index = 0
55    for file_to_copy in files_to_copy:
56      if not os.path.isdir(dest_dir):
57        command = 'mkdir -p %s' % dest_dir
58        self.ce.RunCommand(command)
59      dest_file = os.path.join(dest_dir,
60                               ('%s.%s' % (os.path.basename(file_to_copy),
61                                           file_index)))
62      ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False)
63      if ret:
64        raise Exception('Could not copy results file: %s' % file_to_copy)
65
66  def CopyResultsTo(self, dest_dir):
67    self.CopyFilesTo(dest_dir, self.perf_data_files)
68    self.CopyFilesTo(dest_dir, self.perf_report_files)
69    if len(self.perf_data_files) or len(self.perf_report_files):
70      self._logger.LogOutput('Perf results files stored in %s.' % dest_dir)
71
72  def GetNewKeyvals(self, keyvals_dict):
73    # Initialize 'units' dictionary.
74    units_dict = {}
75    for k in keyvals_dict:
76      units_dict[k] = ''
77    results_files = self.GetDataMeasurementsFiles()
78    for f in results_files:
79      # Make sure we can find the results file
80      if os.path.exists(f):
81        data_filename = f
82      else:
83        # Otherwise get the base filename and create the correct
84        # path for it.
85        f_dir, f_base = misc.GetRoot(f)
86        data_filename = os.path.join(self.chromeos_root, 'chroot/tmp',
87                                     self.temp_dir, f_base)
88      if os.path.exists(data_filename):
89        with open(data_filename, 'r') as data_file:
90          lines = data_file.readlines()
91          for line in lines:
92            tmp_dict = json.loads(line)
93            graph_name = tmp_dict['graph']
94            graph_str = (graph_name + '__') if graph_name else ''
95            key = graph_str + tmp_dict['description']
96            keyvals_dict[key] = tmp_dict['value']
97            units_dict[key] = tmp_dict['units']
98
99    return keyvals_dict, units_dict
100
101  def AppendTelemetryUnits(self, keyvals_dict, units_dict):
102    """keyvals_dict is the dictionary of key-value pairs that is used for generating Crosperf reports.
103
104    units_dict is a dictionary of the units for the return values in
105    keyvals_dict.  We need to associate the units with the return values,
106    for Telemetry tests, so that we can include the units in the reports.
107    This function takes each value in keyvals_dict, finds the corresponding
108    unit in the units_dict, and replaces the old value with a list of the
109    old value and the units.  This later gets properly parsed in the
110    ResultOrganizer class, for generating the reports.
111
112    """
113
114    results_dict = {}
115    for k in keyvals_dict:
116      # We don't want these lines in our reports; they add no useful data.
117      if k == '' or k == 'telemetry_Crosperf':
118        continue
119      val = keyvals_dict[k]
120      units = units_dict[k]
121      new_val = [val, units]
122      results_dict[k] = new_val
123    return results_dict
124
125  def GetKeyvals(self, show_all):
126    results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp')
127    if not self.temp_dir:
128      self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
129      command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir)
130      self.ce.RunCommand(command, print_to_console=False)
131
132    command = ('python generate_test_report --no-color --csv %s' %
133               (os.path.join('/tmp', os.path.basename(self.temp_dir))))
134    _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root,
135                                                 command,
136                                                 print_to_console=False)
137    keyvals_dict = {}
138    tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root,
139                                                 self.temp_dir)
140    for line in out.splitlines():
141      tokens = re.split('=|,', line)
142      key = tokens[-2]
143      if key.startswith(tmp_dir_in_chroot):
144        key = key[len(tmp_dir_in_chroot) + 1:]
145      value = tokens[-1]
146      keyvals_dict[key] = value
147
148    # Check to see if there is a perf_measurements file and get the
149    # data from it if so.
150    keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict)
151    if self.suite == 'telemetry_Crosperf':
152      # For telemtry_Crosperf results, append the units to the return
153      # results, for use in generating the reports.
154      keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict)
155    return keyvals_dict
156
157  def GetResultsDir(self):
158    mo = re.search(r'Results placed in (\S+)', self.out)
159    if mo:
160      result = mo.group(1)
161      return result
162    raise Exception('Could not find results directory.')
163
164  def FindFilesInResultsDir(self, find_args):
165    if not self.results_dir:
166      return None
167
168    command = 'find %s %s' % (self.results_dir, find_args)
169    ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False)
170    if ret:
171      raise Exception('Could not run find command!')
172    return out
173
174  def GetPerfDataFiles(self):
175    return self.FindFilesInResultsDir('-name perf.data').splitlines()
176
177  def GetPerfReportFiles(self):
178    return self.FindFilesInResultsDir('-name perf.data.report').splitlines()
179
180  def GetDataMeasurementsFiles(self):
181    return self.FindFilesInResultsDir('-name perf_measurements').splitlines()
182
183  def GeneratePerfReportFiles(self):
184    perf_report_files = []
185    for perf_data_file in self.perf_data_files:
186      # Generate a perf.report and store it side-by-side with the perf.data
187      # file.
188      chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root,
189                                                       perf_data_file)
190      perf_report_file = '%s.report' % perf_data_file
191      if os.path.exists(perf_report_file):
192        raise Exception('Perf report file already exists: %s' %
193                        perf_report_file)
194      chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root,
195                                                         perf_report_file)
196      perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf')
197
198      perf_file = '/usr/sbin/perf'
199      if os.path.exists(perf_path):
200        perf_file = '/usr/bin/perf'
201
202      # The following is a hack, to use the perf.static binary that
203      # was given to us by Stephane Eranian, until he can figure out
204      # why "normal" perf cannot properly symbolize ChromeOS perf.data files.
205      # Get the directory containing the 'crosperf' script.
206      dirname, _ = misc.GetRoot(sys.argv[0])
207      perf_path = os.path.join(dirname, '..', 'perf.static')
208      if os.path.exists(perf_path):
209        # copy the executable into the chroot so that it can be found.
210        src_path = perf_path
211        dst_path = os.path.join(self.chromeos_root, 'chroot',
212                                'tmp/perf.static')
213        command = 'cp %s %s' % (src_path, dst_path)
214        self.ce.RunCommand(command)
215        perf_file = '/tmp/perf.static'
216
217      command = ('%s report '
218                 '-n '
219                 '--symfs /build/%s '
220                 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux '
221                 '--kallsyms /build/%s/boot/System.map-* '
222                 '-i %s --stdio '
223                 '> %s' % (perf_file, self.board, self.board, self.board,
224                           chroot_perf_data_file, chroot_perf_report_file))
225      self.ce.ChrootRunCommand(self.chromeos_root, command)
226
227      # Add a keyval to the dictionary for the events captured.
228      perf_report_files.append(misc.GetOutsideChrootPath(
229          self.chromeos_root, chroot_perf_report_file))
230    return perf_report_files
231
232  def GatherPerfResults(self):
233    report_id = 0
234    for perf_report_file in self.perf_report_files:
235      with open(perf_report_file, 'r') as f:
236        report_contents = f.read()
237        for group in re.findall(r'Events: (\S+) (\S+)', report_contents):
238          num_events = group[0]
239          event_name = group[1]
240          key = 'perf_%s_%s' % (report_id, event_name)
241          value = str(misc.UnitToNumber(num_events))
242          self.keyvals[key] = value
243
244  def PopulateFromRun(self, out, err, retval, show_all, test, suite):
245    self.board = self.label.board
246    self.out = out
247    self.err = err
248    self.retval = retval
249    self.test_name = test
250    self.suite = suite
251    self.chroot_results_dir = self.GetResultsDir()
252    self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root,
253                                                 self.chroot_results_dir)
254    self.perf_data_files = self.GetPerfDataFiles()
255    # Include all perf.report data in table.
256    self.perf_report_files = self.GeneratePerfReportFiles()
257    # TODO(asharif): Do something similar with perf stat.
258
259    # Grab keyvals from the directory.
260    self.ProcessResults(show_all)
261
262  def ProcessResults(self, show_all):
263    # Note that this function doesn't know anything about whether there is a
264    # cache hit or miss. It should process results agnostic of the cache hit
265    # state.
266    self.keyvals = self.GetKeyvals(show_all)
267    self.keyvals['retval'] = self.retval
268    # Generate report from all perf.data files.
269    # Now parse all perf report files and include them in keyvals.
270    self.GatherPerfResults()
271
272  def PopulateFromCacheDir(self, cache_dir, show_all, test, suite):
273    self.test_name = test
274    self.suite = suite
275    # Read in everything from the cache directory.
276    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
277      self.out = pickle.load(f)
278      self.err = pickle.load(f)
279      self.retval = pickle.load(f)
280
281    # Untar the tarball to a temporary directory
282    self.temp_dir = tempfile.mkdtemp(
283        dir=os.path.join(self.chromeos_root, 'chroot', 'tmp'))
284
285    command = ('cd %s && tar xf %s' %
286               (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL)))
287    ret = self.ce.RunCommand(command, print_to_console=False)
288    if ret:
289      raise Exception('Could not untar cached tarball')
290    self.results_dir = self.temp_dir
291    self.perf_data_files = self.GetPerfDataFiles()
292    self.perf_report_files = self.GetPerfReportFiles()
293    self.ProcessResults(show_all)
294
295  def CleanUp(self, rm_chroot_tmp):
296    if rm_chroot_tmp and self.results_dir:
297      dirname, basename = misc.GetRoot(self.results_dir)
298      if basename.find('test_that_results_') != -1:
299        command = 'rm -rf %s' % self.results_dir
300      else:
301        command = 'rm -rf %s' % dirname
302      self.ce.RunCommand(command)
303    if self.temp_dir:
304      command = 'rm -rf %s' % self.temp_dir
305      self.ce.RunCommand(command)
306
307  def StoreToCacheDir(self, cache_dir, machine_manager, key_list):
308    # Create the dir if it doesn't exist.
309    temp_dir = tempfile.mkdtemp()
310
311    # Store to the temp directory.
312    with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f:
313      pickle.dump(self.out, f)
314      pickle.dump(self.err, f)
315      pickle.dump(self.retval, f)
316
317    if not test_flag.GetTestMode():
318      with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f:
319        f.write('%s\n' % self.label.name)
320        f.write('%s\n' % self.label.chrome_version)
321        f.write('%s\n' % self.machine.checksum_string)
322        for k in key_list:
323          f.write(k)
324          f.write('\n')
325
326    if self.results_dir:
327      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
328      command = ('cd %s && '
329                 'tar '
330                 '--exclude=var/spool '
331                 '--exclude=var/log '
332                 '-cjf %s .' % (self.results_dir, tarball))
333      ret = self.ce.RunCommand(command)
334      if ret:
335        raise Exception("Couldn't store autotest output directory.")
336    # Store machine info.
337    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
338    # this function.
339    with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f:
340      f.write(machine_manager.machine_checksum_string[self.label.name])
341
342    if os.path.exists(cache_dir):
343      command = 'rm -rf {0}'.format(cache_dir)
344      self.ce.RunCommand(command)
345
346    command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir))
347    command += 'chmod g+x {0} && '.format(temp_dir)
348    command += 'mv {0} {1}'.format(temp_dir, cache_dir)
349    ret = self.ce.RunCommand(command)
350    if ret:
351      command = 'rm -rf {0}'.format(temp_dir)
352      self.ce.RunCommand(command)
353      raise Exception('Could not move dir %s to dir %s' % (temp_dir, cache_dir))
354
355  @classmethod
356  def CreateFromRun(cls,
357                    logger,
358                    log_level,
359                    label,
360                    machine,
361                    out,
362                    err,
363                    retval,
364                    show_all,
365                    test,
366                    suite='telemetry_Crosperf'):
367    if suite == 'telemetry':
368      result = TelemetryResult(logger, label, log_level, machine)
369    else:
370      result = cls(logger, label, log_level, machine)
371    result.PopulateFromRun(out, err, retval, show_all, test, suite)
372    return result
373
374  @classmethod
375  def CreateFromCacheHit(cls,
376                         logger,
377                         log_level,
378                         label,
379                         machine,
380                         cache_dir,
381                         show_all,
382                         test,
383                         suite='telemetry_Crosperf'):
384    if suite == 'telemetry':
385      result = TelemetryResult(logger, label, log_level, machine)
386    else:
387      result = cls(logger, label, log_level, machine)
388    try:
389      result.PopulateFromCacheDir(cache_dir, show_all, test, suite)
390
391    except Exception as e:
392      logger.LogError('Exception while using cache: %s' % e)
393      return None
394    return result
395
396
397class TelemetryResult(Result):
398
399  def __init__(self, logger, label, log_level, machine, cmd_exec=None):
400    super(TelemetryResult, self).__init__(logger, label, log_level, machine,
401                                          cmd_exec)
402
403  def PopulateFromRun(self, out, err, retval, show_all, test, suite):
404    self.out = out
405    self.err = err
406    self.retval = retval
407
408    self.ProcessResults()
409
410  def ProcessResults(self):
411    # The output is:
412    # url,average_commit_time (ms),...
413    # www.google.com,33.4,21.2,...
414    # We need to convert to this format:
415    # {"www.google.com:average_commit_time (ms)": "33.4",
416    #  "www.google.com:...": "21.2"}
417    # Added note:  Occasionally the output comes back
418    # with "JSON.stringify(window.automation.GetResults())" on
419    # the first line, and then the rest of the output as
420    # described above.
421
422    lines = self.out.splitlines()
423    self.keyvals = {}
424
425    if lines:
426      if lines[0].startswith('JSON.stringify'):
427        lines = lines[1:]
428
429    if not lines:
430      return
431    labels = lines[0].split(',')
432    for line in lines[1:]:
433      fields = line.split(',')
434      if len(fields) != len(labels):
435        continue
436      for i in range(1, len(labels)):
437        key = '%s %s' % (fields[0], labels[i])
438        value = fields[i]
439        self.keyvals[key] = value
440    self.keyvals['retval'] = self.retval
441
442  def PopulateFromCacheDir(self, cache_dir):
443    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
444      self.out = pickle.load(f)
445      self.err = pickle.load(f)
446      self.retval = pickle.load(f)
447    self.ProcessResults()
448
449
450class CacheConditions(object):
451  # Cache hit only if the result file exists.
452  CACHE_FILE_EXISTS = 0
453
454  # Cache hit if the checksum of cpuinfo and totalmem of
455  # the cached result and the new run match.
456  MACHINES_MATCH = 1
457
458  # Cache hit if the image checksum of the cached result and the new run match.
459  CHECKSUMS_MATCH = 2
460
461  # Cache hit only if the cached result was successful
462  RUN_SUCCEEDED = 3
463
464  # Never a cache hit.
465  FALSE = 4
466
467  # Cache hit if the image path matches the cached image path.
468  IMAGE_PATH_MATCH = 5
469
470  # Cache hit if the uuid of hard disk mataches the cached one
471
472  SAME_MACHINE_MATCH = 6
473
474
475class ResultsCache(object):
476  """ This class manages the key of the cached runs without worrying about what
477  is exactly stored (value). The value generation is handled by the Results
478  class.
479  """
480  CACHE_VERSION = 6
481
482  def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args,
483           profiler_args, machine_manager, machine, board, cache_conditions,
484           logger_to_use, log_level, label, share_cache, suite,
485           show_all_results, run_local):
486    self.chromeos_image = chromeos_image
487    self.chromeos_root = chromeos_root
488    self.test_name = test_name
489    self.iteration = iteration
490    self.test_args = test_args
491    self.profiler_args = profiler_args
492    self.board = board
493    self.cache_conditions = cache_conditions
494    self.machine_manager = machine_manager
495    self.machine = machine
496    self._logger = logger_to_use
497    self.ce = command_executer.GetCommandExecuter(self._logger,
498                                                   log_level=log_level)
499    self.label = label
500    self.share_cache = share_cache
501    self.suite = suite
502    self.log_level = log_level
503    self.show_all = show_all_results
504    self.run_local = run_local
505
506  def GetCacheDirForRead(self):
507    matching_dirs = []
508    for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)):
509      matching_dirs += glob.glob(glob_path)
510
511    if matching_dirs:
512      # Cache file found.
513      return matching_dirs[0]
514    else:
515      return None
516
517  def GetCacheDirForWrite(self, get_keylist=False):
518    cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0]
519    if get_keylist:
520      args_str = '%s_%s_%s' % (self.test_args, self.profiler_args,
521                               self.run_local)
522      version, image = results_report.ParseChromeosImage(
523          self.label.chromeos_image)
524      keylist = [version, image, self.label.board, self.machine.name,
525                 self.test_name, str(self.iteration), args_str]
526      return cache_path, keylist
527    return cache_path
528
529  def FormCacheDir(self, list_of_strings):
530    cache_key = ' '.join(list_of_strings)
531    cache_dir = misc.GetFilenameFromString(cache_key)
532    if self.label.cache_dir:
533      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
534      cache_path = [os.path.join(cache_home, cache_dir)]
535    else:
536      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
537
538    if len(self.share_cache):
539      for path in [x.strip() for x in self.share_cache.split(',')]:
540        if os.path.exists(path):
541          cache_path.append(os.path.join(path, cache_dir))
542        else:
543          self._logger.LogFatal('Unable to find shared cache: %s' % path)
544
545    return cache_path
546
547  def GetCacheKeyList(self, read):
548    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
549      machine_checksum = '*'
550    else:
551      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
552    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
553      checksum = '*'
554    elif self.label.image_type == 'trybot':
555      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
556    elif self.label.image_type == 'official':
557      checksum = '*'
558    else:
559      checksum = ImageChecksummer().Checksum(self.label, self.log_level)
560
561    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
562      image_path_checksum = '*'
563    else:
564      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
565
566    machine_id_checksum = ''
567    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
568      machine_id_checksum = '*'
569    else:
570      if self.machine and self.machine.name in self.label.remote:
571        machine_id_checksum = self.machine.machine_id_checksum
572      else:
573        for machine in self.machine_manager.GetMachines(self.label):
574          if machine.name == self.label.remote[0]:
575            machine_id_checksum = machine.machine_id_checksum
576            break
577
578    temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args,
579                                   self.run_local)
580    test_args_checksum = hashlib.md5(''.join(temp_test_args)).hexdigest()
581    return (image_path_checksum, self.test_name, str(self.iteration),
582            test_args_checksum, checksum, machine_checksum, machine_id_checksum,
583            str(self.CACHE_VERSION))
584
585  def ReadResult(self):
586    if CacheConditions.FALSE in self.cache_conditions:
587      cache_dir = self.GetCacheDirForWrite()
588      command = 'rm -rf {0}'.format(cache_dir)
589      self.ce.RunCommand(command)
590      return None
591    cache_dir = self.GetCacheDirForRead()
592
593    if not cache_dir:
594      return None
595
596    if not os.path.isdir(cache_dir):
597      return None
598
599    if self.log_level == 'verbose':
600      self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir)
601    result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label,
602                                       self.machine, cache_dir, self.show_all,
603                                       self.test_name, self.suite)
604    if not result:
605      return None
606
607    if (result.retval == 0 or
608        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
609      return result
610
611    return None
612
613  def StoreResult(self, result):
614    cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True)
615    result.StoreToCacheDir(cache_dir, self.machine_manager, keylist)
616
617
618class MockResultsCache(ResultsCache):
619
620  def Init(self, *args):
621    pass
622
623  def ReadResult(self):
624    return None
625
626  def StoreResult(self, result):
627    pass
628
629
630class MockResult(Result):
631
632  def PopulateFromRun(self, out, err, retval, show_all, test, suite):
633    self.out = out
634    self.err = err
635    self.retval = retval
636