benchmark_run.py revision c454cee542ca459ef9bd87c9f72e81c822caf1e5 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import datetime
8import os
9import threading
10import time
11import traceback
12
13from utils import command_executer
14from utils import timeline
15
16from suite_runner import SuiteRunner
17from results_cache import MockResult
18from results_cache import MockResultsCache
19from results_cache import Result
20from results_cache import ResultsCache
21from results_cache import TelemetryResult
22
23
24STATUS_FAILED = "FAILED"
25STATUS_SUCCEEDED = "SUCCEEDED"
26STATUS_IMAGING = "IMAGING"
27STATUS_RUNNING = "RUNNING"
28STATUS_WAITING = "WAITING"
29STATUS_PENDING = "PENDING"
30
31class BenchmarkRun(threading.Thread):
32  def __init__(self, name, benchmark,
33               label,
34               iteration,
35               cache_conditions,
36               machine_manager,
37               logger_to_use,
38               log_level,
39               share_users):
40    threading.Thread.__init__(self)
41    self.name = name
42    self._logger = logger_to_use
43    self.log_level = log_level
44    self.benchmark = benchmark
45    self.iteration = iteration
46    self.label = label
47    self.result = None
48    self.terminated = False
49    self.retval = None
50    self.run_completed = False
51    self.machine_manager = machine_manager
52    self.suite_runner = SuiteRunner(self._logger, self.log_level)
53    self.machine = None
54    self.cache_conditions = cache_conditions
55    self.runs_complete = 0
56    self.cache_hit = False
57    self.failure_reason = ""
58    self.test_args = benchmark.test_args
59    self.profiler_args = self._GetExtraAutotestArgs()
60    self._ce = command_executer.GetCommandExecuter(self._logger,
61                                                   log_level=self.log_level)
62    self.timeline = timeline.Timeline()
63    self.timeline.Record(STATUS_PENDING)
64    self.share_users = share_users
65
66  def ReadCache(self):
67    # Just use the first machine for running the cached version,
68    # without locking it.
69    self.cache = ResultsCache()
70    self.cache.Init(self.label.chromeos_image,
71                    self.label.chromeos_root,
72                    self.benchmark.test_name,
73                    self.iteration,
74                    self.test_args,
75                    self.profiler_args,
76                    self.machine_manager,
77                    self.label.board,
78                    self.cache_conditions,
79                    self._logger,
80                    self.log_level,
81                    self.label,
82                    self.share_users,
83                    self.benchmark.suite,
84                    self.benchmark.show_all_results
85                   )
86
87    self.result = self.cache.ReadResult()
88    self.cache_hit = (self.result is not None)
89
90  def run(self):
91    try:
92      self.ReadCache()
93
94      if self.result:
95        self._logger.LogOutput("%s: Cache hit." % self.name)
96        self._logger.LogOutput(self.result.out, print_to_console=False)
97        self._logger.LogError(self.result.err, print_to_console=False)
98
99      else:
100        self._logger.LogOutput("%s: No cache hit." % self.name)
101        self.timeline.Record(STATUS_WAITING)
102        # Try to acquire a machine now.
103        self.machine = self.AcquireMachine()
104        self.result = self.RunTest(self.machine)
105
106        self.cache.remote = self.machine.name
107        self.cache.StoreResult(self.result)
108
109      if self.terminated:
110        return
111
112      if not self.result.retval:
113        self.timeline.Record(STATUS_SUCCEEDED)
114      else:
115        if self.timeline.GetLastEvent() != STATUS_FAILED:
116          self.failure_reason = "Return value of test suite was non-zero."
117          self.timeline.Record(STATUS_FAILED)
118
119    except Exception, e:
120      self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
121      traceback.print_exc()
122      if self.timeline.GetLastEvent() != STATUS_FAILED:
123        self.timeline.Record(STATUS_FAILED)
124        self.failure_reason = str(e)
125    finally:
126      if self.machine:
127        if not self.machine.IsReachable():
128          self._logger.LogOutput("Machine %s is not reachable, removing it."
129                                 % self.machine.name)
130          self.machine_manager.RemoveMachine(self.machine.name)
131        self._logger.LogOutput("Releasing machine: %s" % self.machine.name)
132        self.machine_manager.ReleaseMachine(self.machine)
133        self._logger.LogOutput("Released machine: %s" % self.machine.name)
134
135  def Terminate(self):
136    self.terminated = True
137    self.suite_runner.Terminate()
138    if self.timeline.GetLastEvent() != STATUS_FAILED:
139      self.timeline.Record(STATUS_FAILED)
140      self.failure_reason = "Thread terminated."
141
142  def AcquireMachine(self):
143    while True:
144      if self.terminated:
145        raise Exception("Thread terminated while trying to acquire machine.")
146      machine = self.machine_manager.AcquireMachine(self.label.chromeos_image,
147                                                    self.label)
148
149      if machine:
150        self._logger.LogOutput("%s: Machine %s acquired at %s" %
151                               (self.name,
152                                machine.name,
153                                datetime.datetime.now()))
154        break
155      else:
156        sleep_duration = 10
157        time.sleep(sleep_duration)
158    return machine
159
160  def _GetExtraAutotestArgs(self):
161    if self.benchmark.perf_args and self.benchmark.suite == "telemetry":
162      self._logger.LogError("Telemetry benchmark does not support profiler.")
163
164    if self.benchmark.perf_args and self.benchmark.suite == "test_that":
165      self._logger.LogError("test_that does not support profiler.")
166
167    if self.benchmark.perf_args:
168      perf_args_list = self.benchmark.perf_args.split(" ")
169      perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:]
170      perf_args = " ".join(perf_args_list)
171      if not perf_args_list[0] in ["record", "stat"]:
172        raise Exception("perf_args must start with either record or stat")
173      extra_test_args = ["--profiler=custom_perf",
174                             ("--profiler_args='perf_options=\"%s\"'" %
175                              perf_args)]
176      return " ".join(extra_test_args)
177    else:
178      return ""
179
180  def RunTest(self, machine):
181    self.timeline.Record(STATUS_IMAGING)
182    self.machine_manager.ImageMachine(machine,
183                                      self.label)
184    self.timeline.Record(STATUS_RUNNING)
185    [retval, out, err] = self.suite_runner.Run(machine.name,
186                                                  self.label,
187                                                  self.benchmark,
188                                                  self.test_args,
189                                                  self.profiler_args)
190    self.run_completed = True
191    return Result.CreateFromRun(self._logger,
192                                self.log_level,
193                                self.label,
194                                out,
195                                err,
196                                retval,
197                                self.benchmark.show_all_results,
198                                self.benchmark.test_name,
199                                self.benchmark.suite)
200
201  def SetCacheConditions(self, cache_conditions):
202    self.cache_conditions = cache_conditions
203
204
205class MockBenchmarkRun(BenchmarkRun):
206  """Inherited from BenchmarkRuna."""
207
208  def ReadCache(self):
209    # Just use the first machine for running the cached version,
210    # without locking it.
211    self.cache = MockResultsCache()
212    self.cache.Init(self.label.chromeos_image,
213                    self.label.chromeos_root,
214                    self.benchmark.test_name,
215                    self.iteration,
216                    self.test_args,
217                    self.profiler_args,
218                    self.machine_manager,
219                    self.label.board,
220                    self.cache_conditions,
221                    self._logger,
222                    self.log_level,
223                    self.label,
224                    self.share_users,
225                    self.benchmark.suite,
226                    self.benchmark.show_all_results
227                   )
228
229    self.result = self.cache.ReadResult()
230    self.cache_hit = (self.result is not None)
231
232
233  def RunTest(self, machine):
234    """Remove Result.CreateFromRun for testing."""
235    self.timeline.Record(STATUS_IMAGING)
236    self.machine_manager.ImageMachine(machine,
237                                      self.label)
238    self.timeline.Record(STATUS_RUNNING)
239    [retval, out, err] = self.suite_runner.Run(machine.name,
240                                                  self.label.chromeos_root,
241                                                  self.label.board,
242                                                  self.benchmark.test_name,
243                                                  self.test_args,
244                                                  self.profiler_args)
245    self.run_completed = True
246    rr = MockResult("logger", self.label, self.log_level)
247    rr.out = out
248    rr.err = err
249    rr.retval = retval
250    return rr
251