benchmark_run.py revision 9099a788cd7124024559c064e425ed9caef6e0ac
1 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Module of benchmark runs.""" 7from __future__ import print_function 8 9import datetime 10import threading 11import time 12import traceback 13 14from cros_utils import command_executer 15from cros_utils import timeline 16 17from suite_runner import SuiteRunner 18from results_cache import MockResult 19from results_cache import MockResultsCache 20from results_cache import Result 21from results_cache import ResultsCache 22 23STATUS_FAILED = 'FAILED' 24STATUS_SUCCEEDED = 'SUCCEEDED' 25STATUS_IMAGING = 'IMAGING' 26STATUS_RUNNING = 'RUNNING' 27STATUS_WAITING = 'WAITING' 28STATUS_PENDING = 'PENDING' 29 30 31class BenchmarkRun(threading.Thread): 32 """The benchmarkrun class.""" 33 def __init__(self, name, benchmark, label, iteration, cache_conditions, 34 machine_manager, logger_to_use, log_level, share_cache): 35 threading.Thread.__init__(self) 36 self.name = name 37 self._logger = logger_to_use 38 self.log_level = log_level 39 self.benchmark = benchmark 40 self.iteration = iteration 41 self.label = label 42 self.result = None 43 self.terminated = False 44 self.retval = None 45 self.run_completed = False 46 self.machine_manager = machine_manager 47 self.suite_runner = SuiteRunner(self._logger, self.log_level) 48 self.machine = None 49 self.cache_conditions = cache_conditions 50 self.runs_complete = 0 51 self.cache_hit = False 52 self.failure_reason = '' 53 self.test_args = benchmark.test_args 54 self.cache = None 55 self.profiler_args = self.GetExtraAutotestArgs() 56 self._ce = command_executer.GetCommandExecuter(self._logger, 57 log_level=self.log_level) 58 self.timeline = timeline.Timeline() 59 self.timeline.Record(STATUS_PENDING) 60 self.share_cache = share_cache 61 self.cache_has_been_read = False 62 63 # This is used by schedv2. 64 self.owner_thread = None 65 66 def ReadCache(self): 67 # Just use the first machine for running the cached version, 68 # without locking it. 69 self.cache = ResultsCache() 70 self.cache.Init(self.label.chromeos_image, self.label.chromeos_root, 71 self.benchmark.test_name, self.iteration, self.test_args, 72 self.profiler_args, self.machine_manager, self.machine, 73 self.label.board, self.cache_conditions, self._logger, 74 self.log_level, self.label, self.share_cache, 75 self.benchmark.suite, self.benchmark.show_all_results, 76 self.benchmark.run_local) 77 78 self.result = self.cache.ReadResult() 79 self.cache_hit = (self.result is not None) 80 self.cache_has_been_read = True 81 82 def run(self): 83 try: 84 if not self.cache_has_been_read: 85 self.ReadCache() 86 87 if self.result: 88 self._logger.LogOutput('%s: Cache hit.' % self.name) 89 self._logger.LogOutput(self.result.out, print_to_console=False) 90 self._logger.LogError(self.result.err, print_to_console=False) 91 92 elif self.label.cache_only: 93 self._logger.LogOutput('%s: No cache hit.' % self.name) 94 output = '%s: No Cache hit.' % self.name 95 retval = 1 96 err = 'No cache hit.' 97 self.result = Result.CreateFromRun( 98 self._logger, self.log_level, self.label, self.machine, output, err, 99 retval, self.benchmark.test_name, 100 self.benchmark.suite) 101 102 else: 103 self._logger.LogOutput('%s: No cache hit.' % self.name) 104 self.timeline.Record(STATUS_WAITING) 105 # Try to acquire a machine now. 106 self.machine = self.AcquireMachine() 107 self.cache.machine = self.machine 108 self.result = self.RunTest(self.machine) 109 110 self.cache.remote = self.machine.name 111 self.label.chrome_version = self.machine_manager.GetChromeVersion( 112 self.machine) 113 self.cache.StoreResult(self.result) 114 115 if not self.label.chrome_version: 116 if self.machine: 117 self.label.chrome_version = self.machine_manager.GetChromeVersion( 118 self.machine) 119 elif self.result.chrome_version: 120 self.label.chrome_version = self.result.chrome_version 121 122 if self.terminated: 123 return 124 125 if not self.result.retval: 126 self.timeline.Record(STATUS_SUCCEEDED) 127 else: 128 if self.timeline.GetLastEvent() != STATUS_FAILED: 129 self.failure_reason = 'Return value of test suite was non-zero.' 130 self.timeline.Record(STATUS_FAILED) 131 132 except Exception, e: 133 self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) 134 traceback.print_exc() 135 if self.timeline.GetLastEvent() != STATUS_FAILED: 136 self.timeline.Record(STATUS_FAILED) 137 self.failure_reason = str(e) 138 finally: 139 if self.owner_thread is not None: 140 # In schedv2 mode, we do not lock machine locally. So noop here. 141 pass 142 elif self.machine: 143 if not self.machine.IsReachable(): 144 self._logger.LogOutput('Machine %s is not reachable, removing it.' % 145 self.machine.name) 146 self.machine_manager.RemoveMachine(self.machine.name) 147 self._logger.LogOutput('Releasing machine: %s' % self.machine.name) 148 self.machine_manager.ReleaseMachine(self.machine) 149 self._logger.LogOutput('Released machine: %s' % self.machine.name) 150 151 def Terminate(self): 152 self.terminated = True 153 self.suite_runner.Terminate() 154 if self.timeline.GetLastEvent() != STATUS_FAILED: 155 self.timeline.Record(STATUS_FAILED) 156 self.failure_reason = 'Thread terminated.' 157 158 def AcquireMachine(self): 159 if self.owner_thread is not None: 160 # No need to lock machine locally, DutWorker, which is a thread, is 161 # responsible for running br. 162 return self.owner_thread.dut() 163 while True: 164 machine = None 165 if self.terminated: 166 raise RuntimeError('Thread terminated while trying to acquire machine.') 167 168 machine = self.machine_manager.AcquireMachine(self.label) 169 170 if machine: 171 self._logger.LogOutput('%s: Machine %s acquired at %s' % 172 (self.name, machine.name, 173 datetime.datetime.now())) 174 break 175 else: 176 sleep_duration = 10 177 time.sleep(sleep_duration) 178 return machine 179 180 def GetExtraAutotestArgs(self): 181 if self.benchmark.perf_args and self.benchmark.suite == 'telemetry': 182 self._logger.LogError('Telemetry does not support profiler.') 183 self.benchmark.perf_args = '' 184 185 if self.benchmark.perf_args and self.benchmark.suite == 'test_that': 186 self._logger.LogError('test_that does not support profiler.') 187 self.benchmark.perf_args = '' 188 189 if self.benchmark.perf_args: 190 perf_args_list = self.benchmark.perf_args.split(' ') 191 perf_args_list = [perf_args_list[0]] + ['-a'] + perf_args_list[1:] 192 perf_args = ' '.join(perf_args_list) 193 if not perf_args_list[0] in ['record', 'stat']: 194 raise SyntaxError('perf_args must start with either record or stat') 195 extra_test_args = ['--profiler=custom_perf', 196 ("--profiler_args='perf_options=\"%s\"'" % perf_args)] 197 return ' '.join(extra_test_args) 198 else: 199 return '' 200 201 def RunTest(self, machine): 202 self.timeline.Record(STATUS_IMAGING) 203 if self.owner_thread is not None: 204 # In schedv2 mode, do not even call ImageMachine. Machine image is 205 # guarenteed. 206 pass 207 else: 208 self.machine_manager.ImageMachine(machine, self.label) 209 self.timeline.Record(STATUS_RUNNING) 210 [retval, out, err] = self.suite_runner.Run(machine.name, self.label, 211 self.benchmark, self.test_args, 212 self.profiler_args) 213 self.run_completed = True 214 return Result.CreateFromRun(self._logger, self.log_level, self.label, 215 self.machine, out, err, retval, 216 self.benchmark.test_name, self.benchmark.suite) 217 218 def SetCacheConditions(self, cache_conditions): 219 self.cache_conditions = cache_conditions 220 221 def logger(self): 222 """Return the logger, only used by unittest. 223 224 Returns: 225 self._logger 226 """ 227 228 return self._logger 229 230 def __str__(self): 231 """For better debugging.""" 232 233 return 'BenchmarkRun[name="{}"]'.format(self.name) 234 235 236class MockBenchmarkRun(BenchmarkRun): 237 """Inherited from BenchmarkRun.""" 238 239 def ReadCache(self): 240 # Just use the first machine for running the cached version, 241 # without locking it. 242 self.cache = MockResultsCache() 243 self.cache.Init(self.label.chromeos_image, self.label.chromeos_root, 244 self.benchmark.test_name, self.iteration, self.test_args, 245 self.profiler_args, self.machine_manager, self.machine, 246 self.label.board, self.cache_conditions, self._logger, 247 self.log_level, self.label, self.share_cache, 248 self.benchmark.suite, self.benchmark.show_all_results, 249 self.benchmark.run_local) 250 251 self.result = self.cache.ReadResult() 252 self.cache_hit = (self.result is not None) 253 254 def RunTest(self, machine): 255 """Remove Result.CreateFromRun for testing.""" 256 self.timeline.Record(STATUS_IMAGING) 257 self.machine_manager.ImageMachine(machine, self.label) 258 self.timeline.Record(STATUS_RUNNING) 259 [retval, out, err] = self.suite_runner.Run(machine.name, self.label, 260 self.benchmark, self.test_args, 261 self.profiler_args) 262 self.run_completed = True 263 rr = MockResult('logger', self.label, self.log_level, machine) 264 rr.out = out 265 rr.err = err 266 rr.retval = retval 267 return rr 268