results_cache.py revision 882e707d7e4f63c8e64fa554e77303c4800258c7 
1
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Module to deal with result cache."""
6
7from __future__ import print_function
8
9import glob
10import hashlib
11import os
12import pickle
13import re
14import tempfile
15import json
16import sys
17
18from cros_utils import command_executer
19from cros_utils import misc
20
21from image_checksummer import ImageChecksummer
22
23import results_report
24import test_flag
25
26SCRATCH_DIR = os.path.expanduser('~/cros_scratch')
27RESULTS_FILE = 'results.txt'
28MACHINE_FILE = 'machine.txt'
29AUTOTEST_TARBALL = 'autotest.tbz2'
30PERF_RESULTS_FILE = 'perf-results.txt'
31CACHE_KEYS_FILE = 'cache_keys.txt'
32
33
34class Result(object):
35  """Class for holding the results of a single test run.
36
37  This class manages what exactly is stored inside the cache without knowing
38  what the key of the cache is. For runs with perf, it stores perf.data,
39  perf.report, etc. The key generation is handled by the ResultsCache class.
40  """
41
42  def __init__(self, logger, label, log_level, machine, cmd_exec=None):
43    self.chromeos_root = label.chromeos_root
44    self._logger = logger
45    self.ce = cmd_exec or command_executer.GetCommandExecuter(
46        self._logger,
47        log_level=log_level)
48    self.temp_dir = None
49    self.label = label
50    self.results_dir = None
51    self.log_level = log_level
52    self.machine = machine
53    self.perf_data_files = []
54    self.perf_report_files = []
55    self.results_file = []
56    self.chrome_version = ''
57    self.err = None
58    self.chroot_results_dir = ''
59    self.test_name = ''
60    self.keyvals = None
61    self.board = None
62    self.suite = None
63    self.retval = None
64    self.out = None
65
66  def CopyFilesTo(self, dest_dir, files_to_copy):
67    file_index = 0
68    for file_to_copy in files_to_copy:
69      if not os.path.isdir(dest_dir):
70        command = 'mkdir -p %s' % dest_dir
71        self.ce.RunCommand(command)
72      dest_file = os.path.join(dest_dir,
73                               ('%s.%s' % (os.path.basename(file_to_copy),
74                                           file_index)))
75      ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False)
76      if ret:
77        raise IOError('Could not copy results file: %s' % file_to_copy)
78
79  def CopyResultsTo(self, dest_dir):
80    self.CopyFilesTo(dest_dir, self.perf_data_files)
81    self.CopyFilesTo(dest_dir, self.perf_report_files)
82    if len(self.perf_data_files) or len(self.perf_report_files):
83      self._logger.LogOutput('Perf results files stored in %s.' % dest_dir)
84
85  def GetNewKeyvals(self, keyvals_dict):
86    # Initialize 'units' dictionary.
87    units_dict = {}
88    for k in keyvals_dict:
89      units_dict[k] = ''
90    results_files = self.GetDataMeasurementsFiles()
91    for f in results_files:
92      # Make sure we can find the results file
93      if os.path.exists(f):
94        data_filename = f
95      else:
96        # Otherwise get the base filename and create the correct
97        # path for it.
98        _, f_base = misc.GetRoot(f)
99        data_filename = os.path.join(self.chromeos_root, 'chroot/tmp',
100                                     self.temp_dir, f_base)
101      if data_filename.find('.json') > 0:
102        raw_dict = dict()
103        if os.path.exists(data_filename):
104          with open(data_filename, 'r') as data_file:
105            raw_dict = json.load(data_file)
106
107        for k1 in raw_dict:
108          field_dict = raw_dict[k1]
109          for k2 in field_dict:
110            result_dict = field_dict[k2]
111            key = k1 + '__' + k2
112            if 'value' in result_dict:
113              keyvals_dict[key] = result_dict['value']
114            elif 'values' in result_dict:
115              keyvals_dict[key] = result_dict['values']
116            units_dict[key] = result_dict['units']
117      else:
118        if os.path.exists(data_filename):
119          with open(data_filename, 'r') as data_file:
120            lines = data_file.readlines()
121            for line in lines:
122              tmp_dict = json.loads(line)
123              graph_name = tmp_dict['graph']
124              graph_str = (graph_name + '__') if graph_name else ''
125              key = graph_str + tmp_dict['description']
126              keyvals_dict[key] = tmp_dict['value']
127              units_dict[key] = tmp_dict['units']
128
129    return keyvals_dict, units_dict
130
131  def AppendTelemetryUnits(self, keyvals_dict, units_dict):
132    """keyvals_dict is the dict of key-value used to generate Crosperf reports.
133
134    units_dict is a dictionary of the units for the return values in
135    keyvals_dict.  We need to associate the units with the return values,
136    for Telemetry tests, so that we can include the units in the reports.
137    This function takes each value in keyvals_dict, finds the corresponding
138    unit in the units_dict, and replaces the old value with a list of the
139    old value and the units.  This later gets properly parsed in the
140    ResultOrganizer class, for generating the reports.
141    """
142
143    results_dict = {}
144    for k in keyvals_dict:
145      # We don't want these lines in our reports; they add no useful data.
146      if k == '' or k == 'telemetry_Crosperf':
147        continue
148      val = keyvals_dict[k]
149      units = units_dict[k]
150      new_val = [val, units]
151      results_dict[k] = new_val
152    return results_dict
153
154  def GetKeyvals(self):
155    results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp')
156    if not self.temp_dir:
157      self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
158      command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir)
159      self.ce.RunCommand(command, print_to_console=False)
160
161    command = ('python generate_test_report --no-color --csv %s' %
162               (os.path.join('/tmp', os.path.basename(self.temp_dir))))
163    _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root,
164                                                command,
165                                                print_to_console=False)
166    keyvals_dict = {}
167    tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root,
168                                                 self.temp_dir)
169    for line in out.splitlines():
170      tokens = re.split('=|,', line)
171      key = tokens[-2]
172      if key.startswith(tmp_dir_in_chroot):
173        key = key[len(tmp_dir_in_chroot) + 1:]
174      value = tokens[-1]
175      keyvals_dict[key] = value
176
177    # Check to see if there is a perf_measurements file and get the
178    # data from it if so.
179    keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict)
180    if self.suite == 'telemetry_Crosperf':
181      # For telemtry_Crosperf results, append the units to the return
182      # results, for use in generating the reports.
183      keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict)
184    return keyvals_dict
185
186  def GetResultsDir(self):
187    mo = re.search(r'Results placed in (\S+)', self.out)
188    if mo:
189      result = mo.group(1)
190      return result
191    raise RuntimeError('Could not find results directory.')
192
193  def FindFilesInResultsDir(self, find_args):
194    if not self.results_dir:
195      return None
196
197    command = 'find %s %s' % (self.results_dir, find_args)
198    ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False)
199    if ret:
200      raise RuntimeError('Could not run find command!')
201    return out
202
203  def GetResultsFile(self):
204    return self.FindFilesInResultsDir('-name results-chart.json').splitlines()
205
206  def GetPerfDataFiles(self):
207    return self.FindFilesInResultsDir('-name perf.data').splitlines()
208
209  def GetPerfReportFiles(self):
210    return self.FindFilesInResultsDir('-name perf.data.report').splitlines()
211
212  def GetDataMeasurementsFiles(self):
213    result = self.FindFilesInResultsDir('-name perf_measurements').splitlines()
214    if not result:
215      result = \
216          self.FindFilesInResultsDir('-name results-chart.json').splitlines()
217    return result
218
219  def GeneratePerfReportFiles(self):
220    perf_report_files = []
221    for perf_data_file in self.perf_data_files:
222      # Generate a perf.report and store it side-by-side with the perf.data
223      # file.
224      chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root,
225                                                       perf_data_file)
226      perf_report_file = '%s.report' % perf_data_file
227      if os.path.exists(perf_report_file):
228        raise RuntimeError('Perf report file already exists: %s' %
229                           perf_report_file)
230      chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root,
231                                                         perf_report_file)
232      perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf')
233
234      perf_file = '/usr/sbin/perf'
235      if os.path.exists(perf_path):
236        perf_file = '/usr/bin/perf'
237
238      # The following is a hack, to use the perf.static binary that
239      # was given to us by Stephane Eranian, until he can figure out
240      # why "normal" perf cannot properly symbolize ChromeOS perf.data files.
241      # Get the directory containing the 'crosperf' script.
242      dirname, _ = misc.GetRoot(sys.argv[0])
243      perf_path = os.path.join(dirname, '..', 'perf.static')
244      if os.path.exists(perf_path):
245        # copy the executable into the chroot so that it can be found.
246        src_path = perf_path
247        dst_path = os.path.join(self.chromeos_root, 'chroot',
248                                'tmp/perf.static')
249        command = 'cp %s %s' % (src_path, dst_path)
250        self.ce.RunCommand(command)
251        perf_file = '/tmp/perf.static'
252
253      command = ('%s report '
254                 '-n '
255                 '--symfs /build/%s '
256                 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux '
257                 '--kallsyms /build/%s/boot/System.map-* '
258                 '-i %s --stdio '
259                 '> %s' % (perf_file, self.board, self.board, self.board,
260                           chroot_perf_data_file, chroot_perf_report_file))
261      self.ce.ChrootRunCommand(self.chromeos_root, command)
262
263      # Add a keyval to the dictionary for the events captured.
264      perf_report_files.append(misc.GetOutsideChrootPath(
265          self.chromeos_root, chroot_perf_report_file))
266    return perf_report_files
267
268  def GatherPerfResults(self):
269    report_id = 0
270    for perf_report_file in self.perf_report_files:
271      with open(perf_report_file, 'r') as f:
272        report_contents = f.read()
273        for group in re.findall(r'Events: (\S+) (\S+)', report_contents):
274          num_events = group[0]
275          event_name = group[1]
276          key = 'perf_%s_%s' % (report_id, event_name)
277          value = str(misc.UnitToNumber(num_events))
278          self.keyvals[key] = value
279
280  def PopulateFromRun(self, out, err, retval, test, suite):
281    self.board = self.label.board
282    self.out = out
283    self.err = err
284    self.retval = retval
285    self.test_name = test
286    self.suite = suite
287    self.chroot_results_dir = self.GetResultsDir()
288    self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root,
289                                                 self.chroot_results_dir)
290    self.results_file = self.GetResultsFile()
291    self.perf_data_files = self.GetPerfDataFiles()
292    # Include all perf.report data in table.
293    self.perf_report_files = self.GeneratePerfReportFiles()
294    # TODO(asharif): Do something similar with perf stat.
295
296    # Grab keyvals from the directory.
297    self.ProcessResults()
298
299  def ProcessJsonResults(self):
300    # Open and parse the json results file generated by telemetry/test_that.
301    if not self.results_file:
302      raise IOError('No results file found.')
303    filename = self.results_file[0]
304    if not filename.endswith('.json'):
305      raise IOError('Attempt to call json on non-json file: %s' % filename)
306
307    if not os.path.exists(filename):
308      return {}
309
310    keyvals = {}
311    with open(filename, 'r') as f:
312      raw_dict = json.load(f)
313      if 'charts' in raw_dict:
314        raw_dict = raw_dict['charts']
315      for k, field_dict in raw_dict.iteritems():
316        for item in field_dict:
317          keyname = k + "__" + item
318          value_dict = field_dict[item]
319          if 'value' in value_dict:
320            result = value_dict['value']
321          elif 'values' in value_dict:
322            if not value_dict['values']:
323              continue
324            result = value_dict['values']
325          units = value_dict['units']
326          new_value = [result, units]
327          keyvals[keyname] = new_value
328    return keyvals
329
330  def ProcessResults(self):
331    # Note that this function doesn't know anything about whether there is a
332    # cache hit or miss. It should process results agnostic of the cache hit
333    # state.
334    if self.results_file and self.results_file[0].find('results_chart.json'):
335      self.keyvals = self.ProcessJsonResults()
336    else:
337      print('\n ** WARNING **: Had to use deprecated output-method to '
338            'collect results.\n')
339      self.keyvals = self.GetKeyvals()
340    self.keyvals['retval'] = self.retval
341    # Generate report from all perf.data files.
342    # Now parse all perf report files and include them in keyvals.
343    self.GatherPerfResults()
344
345  def GetChromeVersionFromCache(self, cache_dir):
346    # Read chrome_version from keys file, if present.
347    chrome_version = ''
348    keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE)
349    if os.path.exists(keys_file):
350      with open(keys_file, 'r') as f:
351        lines = f.readlines()
352        for l in lines:
353          if l.startswith('Google Chrome '):
354            chrome_version = l
355            if chrome_version.endswith('\n'):
356              chrome_version = chrome_version[:-1]
357            break
358    return chrome_version
359
360  def PopulateFromCacheDir(self, cache_dir, test, suite):
361    self.test_name = test
362    self.suite = suite
363    # Read in everything from the cache directory.
364    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
365      self.out = pickle.load(f)
366      self.err = pickle.load(f)
367      self.retval = pickle.load(f)
368
369    # Untar the tarball to a temporary directory
370    self.temp_dir = tempfile.mkdtemp(
371        dir=os.path.join(self.chromeos_root, 'chroot', 'tmp'))
372
373    command = ('cd %s && tar xf %s' %
374               (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL)))
375    ret = self.ce.RunCommand(command, print_to_console=False)
376    if ret:
377      raise RuntimeError('Could not untar cached tarball')
378    self.results_dir = self.temp_dir
379    self.perf_data_files = self.GetPerfDataFiles()
380    self.perf_report_files = self.GetPerfReportFiles()
381    self.chrome_version = self.GetChromeVersionFromCache(cache_dir)
382    self.ProcessResults()
383
384  def CleanUp(self, rm_chroot_tmp):
385    if rm_chroot_tmp and self.results_dir:
386      dirname, basename = misc.GetRoot(self.results_dir)
387      if basename.find('test_that_results_') != -1:
388        command = 'rm -rf %s' % self.results_dir
389      else:
390        command = 'rm -rf %s' % dirname
391      self.ce.RunCommand(command)
392    if self.temp_dir:
393      command = 'rm -rf %s' % self.temp_dir
394      self.ce.RunCommand(command)
395
396  def StoreToCacheDir(self, cache_dir, machine_manager, key_list):
397    # Create the dir if it doesn't exist.
398    temp_dir = tempfile.mkdtemp()
399
400    # Store to the temp directory.
401    with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f:
402      pickle.dump(self.out, f)
403      pickle.dump(self.err, f)
404      pickle.dump(self.retval, f)
405
406    if not test_flag.GetTestMode():
407      with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f:
408        f.write('%s\n' % self.label.name)
409        f.write('%s\n' % self.label.chrome_version)
410        f.write('%s\n' % self.machine.checksum_string)
411        for k in key_list:
412          f.write(k)
413          f.write('\n')
414
415    if self.results_dir:
416      tarball = os.path.join(temp_dir, AUTOTEST_TARBALL)
417      command = ('cd %s && '
418                 'tar '
419                 '--exclude=var/spool '
420                 '--exclude=var/log '
421                 '-cjf %s .' % (self.results_dir, tarball))
422      ret = self.ce.RunCommand(command)
423      if ret:
424        raise RuntimeError("Couldn't store autotest output directory.")
425    # Store machine info.
426    # TODO(asharif): Make machine_manager a singleton, and don't pass it into
427    # this function.
428    with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f:
429      f.write(machine_manager.machine_checksum_string[self.label.name])
430
431    if os.path.exists(cache_dir):
432      command = 'rm -rf {0}'.format(cache_dir)
433      self.ce.RunCommand(command)
434
435    command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir))
436    command += 'chmod g+x {0} && '.format(temp_dir)
437    command += 'mv {0} {1}'.format(temp_dir, cache_dir)
438    ret = self.ce.RunCommand(command)
439    if ret:
440      command = 'rm -rf {0}'.format(temp_dir)
441      self.ce.RunCommand(command)
442      raise RuntimeError('Could not move dir %s to dir %s' %
443                         (temp_dir, cache_dir))
444
445  @classmethod
446  def CreateFromRun(cls,
447                    logger,
448                    log_level,
449                    label,
450                    machine,
451                    out,
452                    err,
453                    retval,
454                    test,
455                    suite='telemetry_Crosperf'):
456    if suite == 'telemetry':
457      result = TelemetryResult(logger, label, log_level, machine)
458    else:
459      result = cls(logger, label, log_level, machine)
460    result.PopulateFromRun(out, err, retval, test, suite)
461    return result
462
463  @classmethod
464  def CreateFromCacheHit(cls,
465                         logger,
466                         log_level,
467                         label,
468                         machine,
469                         cache_dir,
470                         test,
471                         suite='telemetry_Crosperf'):
472    if suite == 'telemetry':
473      result = TelemetryResult(logger, label, log_level, machine)
474    else:
475      result = cls(logger, label, log_level, machine)
476    try:
477      result.PopulateFromCacheDir(cache_dir, test, suite)
478
479    except RuntimeError as e:
480      logger.LogError('Exception while using cache: %s' % e)
481      return None
482    return result
483
484
485class TelemetryResult(Result):
486  """Class to hold the results of a single Telemetry run."""
487
488  def __init__(self, logger, label, log_level, machine, cmd_exec=None):
489    super(TelemetryResult, self).__init__(logger, label, log_level, machine,
490                                          cmd_exec)
491
492  def PopulateFromRun(self, out, err, retval, test, suite):
493    self.out = out
494    self.err = err
495    self.retval = retval
496
497    self.ProcessResults()
498
499  def ProcessResults(self):
500    # The output is:
501    # url,average_commit_time (ms),...
502    # www.google.com,33.4,21.2,...
503    # We need to convert to this format:
504    # {"www.google.com:average_commit_time (ms)": "33.4",
505    #  "www.google.com:...": "21.2"}
506    # Added note:  Occasionally the output comes back
507    # with "JSON.stringify(window.automation.GetResults())" on
508    # the first line, and then the rest of the output as
509    # described above.
510
511    lines = self.out.splitlines()
512    self.keyvals = {}
513
514    if lines:
515      if lines[0].startswith('JSON.stringify'):
516        lines = lines[1:]
517
518    if not lines:
519      return
520    labels = lines[0].split(',')
521    for line in lines[1:]:
522      fields = line.split(',')
523      if len(fields) != len(labels):
524        continue
525      for i in xrange(1, len(labels)):
526        key = '%s %s' % (fields[0], labels[i])
527        value = fields[i]
528        self.keyvals[key] = value
529    self.keyvals['retval'] = self.retval
530
531  def PopulateFromCacheDir(self, cache_dir, test, suite):
532    self.test_name = test
533    self.suite = suite
534    with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f:
535      self.out = pickle.load(f)
536      self.err = pickle.load(f)
537      self.retval = pickle.load(f)
538
539    self.chrome_version = \
540        super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir)
541    self.ProcessResults()
542
543
544class CacheConditions(object):
545  """Various Cache condition values, for export."""
546
547  # Cache hit only if the result file exists.
548  CACHE_FILE_EXISTS = 0
549
550  # Cache hit if the checksum of cpuinfo and totalmem of
551  # the cached result and the new run match.
552  MACHINES_MATCH = 1
553
554  # Cache hit if the image checksum of the cached result and the new run match.
555  CHECKSUMS_MATCH = 2
556
557  # Cache hit only if the cached result was successful
558  RUN_SUCCEEDED = 3
559
560  # Never a cache hit.
561  FALSE = 4
562
563  # Cache hit if the image path matches the cached image path.
564  IMAGE_PATH_MATCH = 5
565
566  # Cache hit if the uuid of hard disk mataches the cached one
567
568  SAME_MACHINE_MATCH = 6
569
570
571class ResultsCache(object):
572  """Class to handle the cache for storing/retrieving test run results.
573
574  This class manages the key of the cached runs without worrying about what
575  is exactly stored (value). The value generation is handled by the Results
576  class.
577  """
578  CACHE_VERSION = 6
579
580  def __init__(self):
581    # Proper initialization happens in the Init function below.
582    self.chromeos_image = None
583    self.chromeos_root = None
584    self.test_name = None
585    self.iteration = None
586    self.test_args = None
587    self.profiler_args = None
588    self.board = None
589    self.cache_conditions = None
590    self.machine_manager = None
591    self.machine = None
592    self._logger = None
593    self.ce = None
594    self.label = None
595    self.share_cache = None
596    self.suite = None
597    self.log_level = None
598    self.show_all = None
599    self.run_local = None
600
601
602  def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args,
603           profiler_args, machine_manager, machine, board, cache_conditions,
604           logger_to_use, log_level, label, share_cache, suite,
605           show_all_results, run_local):
606    self.chromeos_image = chromeos_image
607    self.chromeos_root = chromeos_root
608    self.test_name = test_name
609    self.iteration = iteration
610    self.test_args = test_args
611    self.profiler_args = profiler_args
612    self.board = board
613    self.cache_conditions = cache_conditions
614    self.machine_manager = machine_manager
615    self.machine = machine
616    self._logger = logger_to_use
617    self.ce = command_executer.GetCommandExecuter(self._logger,
618                                                  log_level=log_level)
619    self.label = label
620    self.share_cache = share_cache
621    self.suite = suite
622    self.log_level = log_level
623    self.show_all = show_all_results
624    self.run_local = run_local
625
626  def GetCacheDirForRead(self):
627    matching_dirs = []
628    for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)):
629      matching_dirs += glob.glob(glob_path)
630
631    if matching_dirs:
632      # Cache file found.
633      return matching_dirs[0]
634    return None
635
636  def GetCacheDirForWrite(self, get_keylist=False):
637    cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0]
638    if get_keylist:
639      args_str = '%s_%s_%s' % (self.test_args, self.profiler_args,
640                               self.run_local)
641      version, image = results_report.ParseChromeosImage(
642          self.label.chromeos_image)
643      keylist = [version, image, self.label.board, self.machine.name,
644                 self.test_name, str(self.iteration), args_str]
645      return cache_path, keylist
646    return cache_path
647
648  def FormCacheDir(self, list_of_strings):
649    cache_key = ' '.join(list_of_strings)
650    cache_dir = misc.GetFilenameFromString(cache_key)
651    if self.label.cache_dir:
652      cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir))
653      cache_path = [os.path.join(cache_home, cache_dir)]
654    else:
655      cache_path = [os.path.join(SCRATCH_DIR, cache_dir)]
656
657    if len(self.share_cache):
658      for path in [x.strip() for x in self.share_cache.split(',')]:
659        if os.path.exists(path):
660          cache_path.append(os.path.join(path, cache_dir))
661        else:
662          self._logger.LogFatal('Unable to find shared cache: %s' % path)
663
664    return cache_path
665
666  def GetCacheKeyList(self, read):
667    if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions:
668      machine_checksum = '*'
669    else:
670      machine_checksum = self.machine_manager.machine_checksum[self.label.name]
671    if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
672      checksum = '*'
673    elif self.label.image_type == 'trybot':
674      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
675    elif self.label.image_type == 'official':
676      checksum = '*'
677    else:
678      checksum = ImageChecksummer().Checksum(self.label, self.log_level)
679
680    if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
681      image_path_checksum = '*'
682    else:
683      image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest()
684
685    machine_id_checksum = ''
686    if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions:
687      machine_id_checksum = '*'
688    else:
689      if self.machine and self.machine.name in self.label.remote:
690        machine_id_checksum = self.machine.machine_id_checksum
691      else:
692        for machine in self.machine_manager.GetMachines(self.label):
693          if machine.name == self.label.remote[0]:
694            machine_id_checksum = machine.machine_id_checksum
695            break
696
697    temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args,
698                                   self.run_local)
699    test_args_checksum = hashlib.md5(temp_test_args).hexdigest()
700    return (image_path_checksum, self.test_name, str(self.iteration),
701            test_args_checksum, checksum, machine_checksum, machine_id_checksum,
702            str(self.CACHE_VERSION))
703
704  def ReadResult(self):
705    if CacheConditions.FALSE in self.cache_conditions:
706      cache_dir = self.GetCacheDirForWrite()
707      command = 'rm -rf %s' % (cache_dir, )
708      self.ce.RunCommand(command)
709      return None
710    cache_dir = self.GetCacheDirForRead()
711
712    if not cache_dir:
713      return None
714
715    if not os.path.isdir(cache_dir):
716      return None
717
718    if self.log_level == 'verbose':
719      self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir)
720    result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label,
721                                       self.machine, cache_dir,
722                                       self.test_name, self.suite)
723    if not result:
724      return None
725
726    if (result.retval == 0 or
727        CacheConditions.RUN_SUCCEEDED not in self.cache_conditions):
728      return result
729
730    return None
731
732  def StoreResult(self, result):
733    cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True)
734    result.StoreToCacheDir(cache_dir, self.machine_manager, keylist)
735
736
737class MockResultsCache(ResultsCache):
738  """Class for mock testing, corresponding to ResultsCache class."""
739
740  def Init(self, *args):
741    pass
742
743  def ReadResult(self):
744    return None
745
746  def StoreResult(self, result):
747    pass
748
749
750class MockResult(Result):
751  """Class for mock testing, corresponding to Result class."""
752
753  def PopulateFromRun(self, out, err, retval, test, suite):
754    self.out = out
755    self.err = err
756    self.retval = retval
757