benchmark_run.py revision e5a746fb9ba7ddb8827f73f030bf99f798ffcf9e
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import datetime 8import os 9import threading 10import time 11import traceback 12 13from utils import command_executer 14from utils import timeline 15 16from suite_runner import SuiteRunner 17from results_cache import MockResult 18from results_cache import MockResultsCache 19from results_cache import Result 20from results_cache import ResultsCache 21from results_cache import TelemetryResult 22 23 24STATUS_FAILED = "FAILED" 25STATUS_SUCCEEDED = "SUCCEEDED" 26STATUS_IMAGING = "IMAGING" 27STATUS_RUNNING = "RUNNING" 28STATUS_WAITING = "WAITING" 29STATUS_PENDING = "PENDING" 30 31class BenchmarkRun(threading.Thread): 32 def __init__(self, name, benchmark, 33 label, 34 iteration, 35 cache_conditions, 36 machine_manager, 37 logger_to_use, 38 share_users): 39 threading.Thread.__init__(self) 40 self.name = name 41 self._logger = logger_to_use 42 self.benchmark = benchmark 43 self.iteration = iteration 44 self.label = label 45 self.result = None 46 self.terminated = False 47 self.retval = None 48 self.run_completed = False 49 self.machine_manager = machine_manager 50 self.suite_runner = SuiteRunner(self._logger) 51 self.machine = None 52 self.cache_conditions = cache_conditions 53 self.runs_complete = 0 54 self.cache_hit = False 55 self.failure_reason = "" 56 self.test_args = benchmark.test_args 57 self.profiler_args = self._GetExtraAutotestArgs() 58 self._ce = command_executer.GetCommandExecuter(self._logger) 59 self.timeline = timeline.Timeline() 60 self.timeline.Record(STATUS_PENDING) 61 self.share_users = share_users 62 63 def ReadCache(self): 64 # Just use the first machine for running the cached version, 65 # without locking it. 66 self.cache = ResultsCache() 67 self.cache.Init(self.label.chromeos_image, 68 self.label.chromeos_root, 69 self.benchmark.test_name, 70 self.iteration, 71 self.test_args, 72 self.profiler_args, 73 self.machine_manager, 74 self.label.board, 75 self.cache_conditions, 76 self._logger, 77 self.label, 78 self.share_users, 79 self.benchmark.suite 80 ) 81 82 self.result = self.cache.ReadResult() 83 self.cache_hit = (self.result is not None) 84 85 def run(self): 86 try: 87 self.ReadCache() 88 89 if self.result: 90 self._logger.LogOutput("%s: Cache hit." % self.name) 91 self._logger.LogOutput(self.result.out, print_to_console=False) 92 self._logger.LogError(self.result.err, print_to_console=False) 93 94 else: 95 self._logger.LogOutput("%s: No cache hit." % self.name) 96 self.timeline.Record(STATUS_WAITING) 97 # Try to acquire a machine now. 98 self.machine = self.AcquireMachine() 99 self.result = self.RunTest(self.machine) 100 101 self.cache.remote = self.machine.name 102 self.cache.StoreResult(self.result) 103 104 if self.terminated: 105 return 106 107 if not self.result.retval: 108 self.timeline.Record(STATUS_SUCCEEDED) 109 else: 110 if self.timeline.GetLastEvent() != STATUS_FAILED: 111 self.failure_reason = "Return value of test suite was non-zero." 112 self.timeline.Record(STATUS_FAILED) 113 114 except Exception, e: 115 self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) 116 traceback.print_exc() 117 if self.timeline.GetLastEvent() != STATUS_FAILED: 118 self.timeline.Record(STATUS_FAILED) 119 self.failure_reason = str(e) 120 finally: 121 if self.machine: 122 if not self.machine.IsReachable(): 123 self._logger.LogOutput("Machine %s is not reachable, removing it." 124 % self.machine.name) 125 self.machine_manager.RemoveMachine(self.machine.name) 126 self._logger.LogOutput("Releasing machine: %s" % self.machine.name) 127 self.machine_manager.ReleaseMachine(self.machine) 128 self._logger.LogOutput("Released machine: %s" % self.machine.name) 129 130 def Terminate(self): 131 self.terminated = True 132 self.suite_runner.Terminate() 133 if self.timeline.GetLastEvent() != STATUS_FAILED: 134 self.timeline.Record(STATUS_FAILED) 135 self.failure_reason = "Thread terminated." 136 137 def AcquireMachine(self): 138 while True: 139 if self.terminated: 140 raise Exception("Thread terminated while trying to acquire machine.") 141 machine = self.machine_manager.AcquireMachine(self.label.chromeos_image, 142 self.label) 143 144 if machine: 145 self._logger.LogOutput("%s: Machine %s acquired at %s" % 146 (self.name, 147 machine.name, 148 datetime.datetime.now())) 149 break 150 else: 151 sleep_duration = 10 152 time.sleep(sleep_duration) 153 return machine 154 155 def _GetExtraAutotestArgs(self): 156 if self.benchmark.perf_args and self.benchmark.suite == "telemetry": 157 self._logger.LogError("Telemetry benchmark does not support profiler.") 158 159 if self.benchmark.perf_args and self.benchmark.suite == "test_that": 160 self._logger.LogError("test_that does not support profiler.") 161 162 if self.benchmark.perf_args: 163 perf_args_list = self.benchmark.perf_args.split(" ") 164 perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:] 165 perf_args = " ".join(perf_args_list) 166 if not perf_args_list[0] in ["record", "stat"]: 167 raise Exception("perf_args must start with either record or stat") 168 extra_test_args = ["--profiler=custom_perf", 169 ("--profiler_args='perf_options=\"%s\"'" % 170 perf_args)] 171 return " ".join(extra_test_args) 172 else: 173 return "" 174 175 def RunTest(self, machine): 176 self.timeline.Record(STATUS_IMAGING) 177 self.machine_manager.ImageMachine(machine, 178 self.label) 179 self.timeline.Record(STATUS_RUNNING) 180 [retval, out, err] = self.suite_runner.Run(machine.name, 181 self.label, 182 self.benchmark, 183 self.test_args, 184 self.profiler_args) 185 self.run_completed = True 186 return Result.CreateFromRun(self._logger, 187 self.label, 188 out, 189 err, 190 retval, 191 self.benchmark.suite) 192 193 def SetCacheConditions(self, cache_conditions): 194 self.cache_conditions = cache_conditions 195 196 197class MockBenchmarkRun(BenchmarkRun): 198 """Inherited from BenchmarkRuna.""" 199 200 def ReadCache(self): 201 # Just use the first machine for running the cached version, 202 # without locking it. 203 self.cache = MockResultsCache() 204 self.cache.Init(self.label.chromeos_image, 205 self.label.chromeos_root, 206 self.benchmark.test_name, 207 self.iteration, 208 self.test_args, 209 self.profiler_args, 210 self.machine_manager, 211 self.label.board, 212 self.cache_conditions, 213 self._logger, 214 self.label, 215 self.share_users, 216 self.benchmark.suite 217 ) 218 219 self.result = self.cache.ReadResult() 220 self.cache_hit = (self.result is not None) 221 222 223 def RunTest(self, machine): 224 """Remove Result.CreateFromRun for testing.""" 225 self.timeline.Record(STATUS_IMAGING) 226 self.machine_manager.ImageMachine(machine, 227 self.label) 228 self.timeline.Record(STATUS_RUNNING) 229 [retval, out, err] = self.suite_runner.Run(machine.name, 230 self.label.chromeos_root, 231 self.label.board, 232 self.benchmark.test_name, 233 self.test_args, 234 self.profiler_args) 235 self.run_completed = True 236 rr = MockResult("logger", self.label) 237 rr.out = out 238 rr.err = err 239 rr.retval = retval 240 return rr 241 242