benchmark_run.py revision 9847df92a2b5f76ccddc4bf10288819712a8ca47 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import datetime
8import os
9import threading
10import time
11import traceback
12
13from utils import command_executer
14from utils import timeline
15
16from suite_runner import SuiteRunner
17from results_cache import MockResult
18from results_cache import MockResultsCache
19from results_cache import Result
20from results_cache import ResultsCache
21from results_cache import TelemetryResult
22
23
24STATUS_FAILED = "FAILED"
25STATUS_SUCCEEDED = "SUCCEEDED"
26STATUS_IMAGING = "IMAGING"
27STATUS_RUNNING = "RUNNING"
28STATUS_WAITING = "WAITING"
29STATUS_PENDING = "PENDING"
30
31class BenchmarkRun(threading.Thread):
32  def __init__(self, name, benchmark,
33               label,
34               iteration,
35               cache_conditions,
36               machine_manager,
37               logger_to_use,
38               share_users):
39    threading.Thread.__init__(self)
40    self.name = name
41    self._logger = logger_to_use
42    self.benchmark = benchmark
43    self.iteration = iteration
44    self.label = label
45    self.result = None
46    self.terminated = False
47    self.retval = None
48    self.run_completed = False
49    self.machine_manager = machine_manager
50    self.suite_runner = SuiteRunner(self._logger)
51    self.machine = None
52    self.cache_conditions = cache_conditions
53    self.runs_complete = 0
54    self.cache_hit = False
55    self.failure_reason = ""
56    self.test_args = benchmark.test_args
57    self.profiler_args = self._GetExtraAutotestArgs()
58    self._ce = command_executer.GetCommandExecuter(self._logger)
59    self.timeline = timeline.Timeline()
60    self.timeline.Record(STATUS_PENDING)
61    self.share_users = share_users
62
63  def ReadCache(self):
64    # Just use the first machine for running the cached version,
65    # without locking it.
66    self.cache = ResultsCache()
67    self.cache.Init(self.label.chromeos_image,
68                    self.label.chromeos_root,
69                    self.benchmark.test_name,
70                    self.iteration,
71                    self.test_args,
72                    self.profiler_args,
73                    self.machine_manager,
74                    self.label.board,
75                    self.cache_conditions,
76                    self._logger,
77                    self.label,
78                    self.share_users,
79                    self.benchmark.suite
80                   )
81
82    self.result = self.cache.ReadResult()
83    self.cache_hit = (self.result is not None)
84
85  def run(self):
86    try:
87      self.ReadCache()
88
89      if self.result:
90        self._logger.LogOutput("%s: Cache hit." % self.name)
91        self._logger.LogOutput(self.result.out, print_to_console=False)
92        self._logger.LogError(self.result.err, print_to_console=False)
93
94      else:
95        self._logger.LogOutput("%s: No cache hit." % self.name)
96        self.timeline.Record(STATUS_WAITING)
97        # Try to acquire a machine now.
98        self.machine = self.AcquireMachine()
99        self.result = self.RunTest(self.machine)
100
101        self.cache.remote = self.machine.name
102        self.cache.StoreResult(self.result)
103
104      if self.terminated:
105        return
106
107      if not self.result.retval:
108        self.timeline.Record(STATUS_SUCCEEDED)
109      else:
110        if self.timeline.GetLastEvent() != STATUS_FAILED:
111          self.failure_reason = "Return value of test suite was non-zero."
112          self.timeline.Record(STATUS_FAILED)
113
114    except Exception, e:
115      self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
116      traceback.print_exc()
117      if self.timeline.GetLastEvent() != STATUS_FAILED:
118        self.timeline.Record(STATUS_FAILED)
119        self.failure_reason = str(e)
120    finally:
121      if self.machine:
122        if not self.machine.IsReachable():
123          self._logger.LogOutput("Machine %s is not reachable, removing it."
124                                 % self.machine.name)
125          self.machine_manager.RemoveMachine(self.machine.name)
126        self._logger.LogOutput("Releasing machine: %s" % self.machine.name)
127        self.machine_manager.ReleaseMachine(self.machine)
128        self._logger.LogOutput("Released machine: %s" % self.machine.name)
129
130  def Terminate(self):
131    self.terminated = True
132    self.suite_runner.Terminate()
133    if self.timeline.GetLastEvent() != STATUS_FAILED:
134      self.timeline.Record(STATUS_FAILED)
135      self.failure_reason = "Thread terminated."
136
137  def AcquireMachine(self):
138    while True:
139      if self.terminated:
140        raise Exception("Thread terminated while trying to acquire machine.")
141      machine = self.machine_manager.AcquireMachine(self.label.chromeos_image,
142                                                    self.label)
143
144      if machine:
145        self._logger.LogOutput("%s: Machine %s acquired at %s" %
146                               (self.name,
147                                machine.name,
148                                datetime.datetime.now()))
149        break
150      else:
151        sleep_duration = 10
152        time.sleep(sleep_duration)
153    return machine
154
155  def _GetExtraAutotestArgs(self):
156    if self.benchmark.perf_args and self.benchmark.suite == "telemetry":
157      self._logger.LogError("Telemetry benchmark does not support profiler.")
158
159    if self.benchmark.perf_args and self.benchmark.suite == "test_that":
160      self._logger.LogError("test_that does not support profiler.")
161
162    if self.benchmark.perf_args:
163      perf_args_list = self.benchmark.perf_args.split(" ")
164      perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:]
165      perf_args = " ".join(perf_args_list)
166      if not perf_args_list[0] in ["record", "stat"]:
167        raise Exception("perf_args must start with either record or stat")
168      extra_test_args = ["--profiler=custom_perf",
169                             ("--profiler_args='perf_options=\"%s\"'" %
170                              perf_args)]
171      return " ".join(extra_test_args)
172    else:
173      return ""
174
175  def RunTest(self, machine):
176    self.timeline.Record(STATUS_IMAGING)
177    self.machine_manager.ImageMachine(machine,
178                                      self.label)
179    self.timeline.Record(STATUS_RUNNING)
180    [retval, out, err] = self.suite_runner.Run(machine.name,
181                                                  self.label,
182                                                  self.benchmark,
183                                                  self.test_args,
184                                                  self.profiler_args)
185    self.run_completed = True
186    return Result.CreateFromRun(self._logger,
187                                self.label,
188                                out,
189                                err,
190                                retval,
191                                self.benchmark.show_all_results,
192                                self.benchmark.test_name,
193                                self.benchmark.suite)
194
195  def SetCacheConditions(self, cache_conditions):
196    self.cache_conditions = cache_conditions
197
198
199class MockBenchmarkRun(BenchmarkRun):
200  """Inherited from BenchmarkRuna."""
201
202  def ReadCache(self):
203    # Just use the first machine for running the cached version,
204    # without locking it.
205    self.cache = MockResultsCache()
206    self.cache.Init(self.label.chromeos_image,
207                    self.label.chromeos_root,
208                    self.benchmark.test_name,
209                    self.iteration,
210                    self.test_args,
211                    self.profiler_args,
212                    self.machine_manager,
213                    self.label.board,
214                    self.cache_conditions,
215                    self._logger,
216                    self.label,
217                    self.share_users,
218                    self.benchmark.suite
219                   )
220
221    self.result = self.cache.ReadResult()
222    self.cache_hit = (self.result is not None)
223
224
225  def RunTest(self, machine):
226    """Remove Result.CreateFromRun for testing."""
227    self.timeline.Record(STATUS_IMAGING)
228    self.machine_manager.ImageMachine(machine,
229                                      self.label)
230    self.timeline.Record(STATUS_RUNNING)
231    [retval, out, err] = self.suite_runner.Run(machine.name,
232                                                  self.label.chromeos_root,
233                                                  self.label.board,
234                                                  self.benchmark.test_name,
235                                                  self.test_args,
236                                                  self.profiler_args)
237    self.run_completed = True
238    rr = MockResult("logger", self.label)
239    rr.out = out
240    rr.err = err
241    rr.retval = retval
242    return rr
243