benchmark_run.py revision 139092449a545d4d87c06af3a6d8f3d71e42e927
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import datetime 8import os 9import threading 10import time 11import traceback 12 13from utils import command_executer 14from utils import timeline 15 16from suite_runner import SuiteRunner 17from results_cache import MockResult 18from results_cache import MockResultsCache 19from results_cache import Result 20from results_cache import ResultsCache 21from results_cache import TelemetryResult 22 23 24STATUS_FAILED = "FAILED" 25STATUS_SUCCEEDED = "SUCCEEDED" 26STATUS_IMAGING = "IMAGING" 27STATUS_RUNNING = "RUNNING" 28STATUS_WAITING = "WAITING" 29STATUS_PENDING = "PENDING" 30 31class BenchmarkRun(threading.Thread): 32 def __init__(self, name, benchmark, 33 label, 34 iteration, 35 cache_conditions, 36 machine_manager, 37 logger_to_use, 38 log_level, 39 share_users): 40 threading.Thread.__init__(self) 41 self.name = name 42 self._logger = logger_to_use 43 self.log_level = log_level 44 self.benchmark = benchmark 45 self.iteration = iteration 46 self.label = label 47 self.result = None 48 self.terminated = False 49 self.retval = None 50 self.run_completed = False 51 self.machine_manager = machine_manager 52 self.suite_runner = SuiteRunner(self._logger, self.log_level) 53 self.machine = None 54 self.cache_conditions = cache_conditions 55 self.runs_complete = 0 56 self.cache_hit = False 57 self.failure_reason = "" 58 self.test_args = benchmark.test_args 59 self.profiler_args = self._GetExtraAutotestArgs() 60 self._ce = command_executer.GetCommandExecuter(self._logger, 61 log_level=self.log_level) 62 self.timeline = timeline.Timeline() 63 self.timeline.Record(STATUS_PENDING) 64 self.share_users = share_users 65 66 def ReadCache(self): 67 # Just use the first machine for running the cached version, 68 # without locking it. 69 self.cache = ResultsCache() 70 self.cache.Init(self.label.chromeos_image, 71 self.label.chromeos_root, 72 self.benchmark.test_name, 73 self.iteration, 74 self.test_args, 75 self.profiler_args, 76 self.machine_manager, 77 self.label.board, 78 self.cache_conditions, 79 self._logger, 80 self.log_level, 81 self.label, 82 self.share_users, 83 self.benchmark.suite, 84 self.benchmark.show_all_results 85 ) 86 87 self.result = self.cache.ReadResult() 88 self.cache_hit = (self.result is not None) 89 90 def run(self): 91 try: 92 self.ReadCache() 93 94 if self.result: 95 self._logger.LogOutput("%s: Cache hit." % self.name) 96 self._logger.LogOutput(self.result.out, print_to_console=False) 97 self._logger.LogError(self.result.err, print_to_console=False) 98 99 else: 100 self._logger.LogOutput("%s: No cache hit." % self.name) 101 self.timeline.Record(STATUS_WAITING) 102 # Try to acquire a machine now. 103 self.machine = self.AcquireMachine() 104 self.result = self.RunTest(self.machine) 105 106 self.cache.remote = self.machine.name 107 self.cache.StoreResult(self.result) 108 109 if self.terminated: 110 return 111 112 if not self.result.retval: 113 self.timeline.Record(STATUS_SUCCEEDED) 114 else: 115 if self.timeline.GetLastEvent() != STATUS_FAILED: 116 self.failure_reason = "Return value of test suite was non-zero." 117 self.timeline.Record(STATUS_FAILED) 118 119 except Exception, e: 120 self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) 121 traceback.print_exc() 122 if self.timeline.GetLastEvent() != STATUS_FAILED: 123 self.timeline.Record(STATUS_FAILED) 124 self.failure_reason = str(e) 125 finally: 126 if self.machine: 127 if not self.machine.IsReachable(): 128 self._logger.LogOutput("Machine %s is not reachable, removing it." 129 % self.machine.name) 130 self.machine_manager.RemoveMachine(self.machine.name) 131 self._logger.LogOutput("Releasing machine: %s" % self.machine.name) 132 self.machine_manager.ReleaseMachine(self.machine) 133 self._logger.LogOutput("Released machine: %s" % self.machine.name) 134 135 def Terminate(self): 136 self.terminated = True 137 self.suite_runner.Terminate() 138 if self.timeline.GetLastEvent() != STATUS_FAILED: 139 self.timeline.Record(STATUS_FAILED) 140 self.failure_reason = "Thread terminated." 141 142 def AcquireMachine(self): 143 while True: 144 if self.terminated: 145 raise Exception("Thread terminated while trying to acquire machine.") 146 machine = self.machine_manager.AcquireMachine(self.label.chromeos_image, 147 self.label) 148 149 if machine: 150 self._logger.LogOutput("%s: Machine %s acquired at %s" % 151 (self.name, 152 machine.name, 153 datetime.datetime.now())) 154 break 155 else: 156 sleep_duration = 10 157 time.sleep(sleep_duration) 158 return machine 159 160 def _GetExtraAutotestArgs(self): 161 if self.benchmark.perf_args and self.benchmark.suite == "telemetry": 162 self._logger.LogError("Telemetry benchmark does not support profiler.") 163 164 if self.benchmark.perf_args and self.benchmark.suite == "test_that": 165 self._logger.LogError("test_that does not support profiler.") 166 167 if self.benchmark.perf_args: 168 perf_args_list = self.benchmark.perf_args.split(" ") 169 perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:] 170 perf_args = " ".join(perf_args_list) 171 if not perf_args_list[0] in ["record", "stat"]: 172 raise Exception("perf_args must start with either record or stat") 173 extra_test_args = ["--profiler=custom_perf", 174 ("--profiler_args='perf_options=\"%s\"'" % 175 perf_args)] 176 return " ".join(extra_test_args) 177 else: 178 return "" 179 180 def RunTest(self, machine): 181 self.timeline.Record(STATUS_IMAGING) 182 self.machine_manager.ImageMachine(machine, 183 self.label) 184 self.timeline.Record(STATUS_RUNNING) 185 [retval, out, err] = self.suite_runner.Run(machine.name, 186 self.label, 187 self.benchmark, 188 self.test_args, 189 self.profiler_args) 190 self.run_completed = True 191 return Result.CreateFromRun(self._logger, 192 self.log_level, 193 self.label, 194 out, 195 err, 196 retval, 197 self.benchmark.show_all_results, 198 self.benchmark.test_name, 199 self.benchmark.suite) 200 201 def SetCacheConditions(self, cache_conditions): 202 self.cache_conditions = cache_conditions 203 204 205class MockBenchmarkRun(BenchmarkRun): 206 """Inherited from BenchmarkRuna.""" 207 208 def ReadCache(self): 209 # Just use the first machine for running the cached version, 210 # without locking it. 211 self.cache = MockResultsCache() 212 self.cache.Init(self.label.chromeos_image, 213 self.label.chromeos_root, 214 self.benchmark.test_name, 215 self.iteration, 216 self.test_args, 217 self.profiler_args, 218 self.machine_manager, 219 self.label.board, 220 self.cache_conditions, 221 self._logger, 222 self.log_level, 223 self.label, 224 self.share_users, 225 self.benchmark.suite, 226 self.benchmark.show_all_results 227 ) 228 229 self.result = self.cache.ReadResult() 230 self.cache_hit = (self.result is not None) 231 232 233 def RunTest(self, machine): 234 """Remove Result.CreateFromRun for testing.""" 235 self.timeline.Record(STATUS_IMAGING) 236 self.machine_manager.ImageMachine(machine, 237 self.label) 238 self.timeline.Record(STATUS_RUNNING) 239 [retval, out, err] = self.suite_runner.Run(machine.name, 240 self.label.chromeos_root, 241 self.label.board, 242 self.benchmark.test_name, 243 self.test_args, 244 self.profiler_args) 245 self.run_completed = True 246 rr = MockResult("logger", self.label) 247 rr.out = out 248 rr.err = err 249 rr.retval = retval 250 return rr 251