benchmark_run.py revision d97422aef5709f0a3b16b4efebc397c891940c95 
1
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Module of benchmark runs."""
7from __future__ import print_function
8
9import datetime
10import threading
11import time
12import traceback
13
14from cros_utils import command_executer
15from cros_utils import timeline
16
17from suite_runner import SuiteRunner
18from results_cache import MockResult
19from results_cache import MockResultsCache
20from results_cache import Result
21from results_cache import ResultsCache
22
23STATUS_FAILED = 'FAILED'
24STATUS_SUCCEEDED = 'SUCCEEDED'
25STATUS_IMAGING = 'IMAGING'
26STATUS_RUNNING = 'RUNNING'
27STATUS_WAITING = 'WAITING'
28STATUS_PENDING = 'PENDING'
29
30
31class BenchmarkRun(threading.Thread):
32  """The benchmarkrun class."""
33  def __init__(self, name, benchmark, label, iteration, cache_conditions,
34               machine_manager, logger_to_use, log_level, share_cache):
35    threading.Thread.__init__(self)
36    self.name = name
37    self._logger = logger_to_use
38    self.log_level = log_level
39    self.benchmark = benchmark
40    self.iteration = iteration
41    self.label = label
42    self.result = None
43    self.terminated = False
44    self.retval = None
45    self.run_completed = False
46    self.machine_manager = machine_manager
47    self.suite_runner = SuiteRunner(self._logger, self.log_level)
48    self.machine = None
49    self.cache_conditions = cache_conditions
50    self.runs_complete = 0
51    self.cache_hit = False
52    self.failure_reason = ''
53    self.test_args = benchmark.test_args
54    self.cache = None
55    self.profiler_args = self._GetExtraAutotestArgs()
56    self._ce = command_executer.GetCommandExecuter(self._logger,
57                                                   log_level=self.log_level)
58    self.timeline = timeline.Timeline()
59    self.timeline.Record(STATUS_PENDING)
60    self.share_cache = share_cache
61    self.cache_has_been_read = False
62
63    # This is used by schedv2.
64    self.owner_thread = None
65
66  def ReadCache(self):
67    # Just use the first machine for running the cached version,
68    # without locking it.
69    self.cache = ResultsCache()
70    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
71                    self.benchmark.test_name, self.iteration, self.test_args,
72                    self.profiler_args, self.machine_manager, self.machine,
73                    self.label.board, self.cache_conditions, self._logger,
74                    self.log_level, self.label, self.share_cache,
75                    self.benchmark.suite, self.benchmark.show_all_results,
76                    self.benchmark.run_local)
77
78    self.result = self.cache.ReadResult()
79    self.cache_hit = (self.result is not None)
80    self.cache_has_been_read = True
81
82  def run(self):
83    try:
84      if not self.cache_has_been_read:
85        self.ReadCache()
86
87      if self.result:
88        self._logger.LogOutput('%s: Cache hit.' % self.name)
89        self._logger.LogOutput(self.result.out, print_to_console=False)
90        self._logger.LogError(self.result.err, print_to_console=False)
91
92      elif self.label.cache_only:
93        self._logger.LogOutput('%s: No cache hit.' % self.name)
94        output = '%s: No Cache hit.' % self.name
95        retval = 1
96        err = 'No cache hit.'
97        self.result = Result.CreateFromRun(
98            self._logger, self.log_level, self.label, self.machine, output, err,
99            retval, self.benchmark.show_all_results, self.benchmark.test_name,
100            self.benchmark.suite)
101
102      else:
103        self._logger.LogOutput('%s: No cache hit.' % self.name)
104        self.timeline.Record(STATUS_WAITING)
105        # Try to acquire a machine now.
106        self.machine = self.AcquireMachine()
107        self.cache.machine = self.machine
108        self.result = self.RunTest(self.machine)
109
110        self.cache.remote = self.machine.name
111        self.label.chrome_version = self.machine_manager.GetChromeVersion(
112            self.machine)
113        self.cache.StoreResult(self.result)
114
115      if self.machine and not self.label.chrome_version:
116        self.label.chrome_version = self.machine_manager.GetChromeVersion(
117            self.machine)
118
119      if self.terminated:
120        return
121
122      if not self.result.retval:
123        self.timeline.Record(STATUS_SUCCEEDED)
124      else:
125        if self.timeline.GetLastEvent() != STATUS_FAILED:
126          self.failure_reason = 'Return value of test suite was non-zero.'
127          self.timeline.Record(STATUS_FAILED)
128
129    except Exception, e:
130      self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
131      traceback.print_exc()
132      if self.timeline.GetLastEvent() != STATUS_FAILED:
133        self.timeline.Record(STATUS_FAILED)
134        self.failure_reason = str(e)
135    finally:
136      if self.owner_thread is not None:
137        # In schedv2 mode, we do not lock machine locally. So noop here.
138        pass
139      elif self.machine:
140        if not self.machine.IsReachable():
141          self._logger.LogOutput('Machine %s is not reachable, removing it.' %
142                                 self.machine.name)
143          self.machine_manager.RemoveMachine(self.machine.name)
144        self._logger.LogOutput('Releasing machine: %s' % self.machine.name)
145        self.machine_manager.ReleaseMachine(self.machine)
146        self._logger.LogOutput('Released machine: %s' % self.machine.name)
147
148  def Terminate(self):
149    self.terminated = True
150    self.suite_runner.Terminate()
151    if self.timeline.GetLastEvent() != STATUS_FAILED:
152      self.timeline.Record(STATUS_FAILED)
153      self.failure_reason = 'Thread terminated.'
154
155  def AcquireMachine(self):
156    if self.owner_thread is not None:
157      # No need to lock machine locally, DutWorker, which is a thread, is
158      # responsible for running br.
159      return self.owner_thread.dut()
160    while True:
161      machine = None
162      if self.terminated:
163        raise Exception('Thread terminated while trying to acquire machine.')
164
165      machine = self.machine_manager.AcquireMachine(self.label)
166
167      if machine:
168        self._logger.LogOutput('%s: Machine %s acquired at %s' %
169                               (self.name, machine.name,
170                                datetime.datetime.now()))
171        break
172      else:
173        sleep_duration = 10
174        time.sleep(sleep_duration)
175    return machine
176
177  def _GetExtraAutotestArgs(self):
178    if self.benchmark.perf_args and self.benchmark.suite == 'telemetry':
179      self._logger.LogError('Telemetry does not support profiler.')
180      self.benchmark.perf_args = ''
181
182    if self.benchmark.perf_args and self.benchmark.suite == 'test_that':
183      self._logger.LogError('test_that does not support profiler.')
184      self.benchmark.perf_args = ''
185
186    if self.benchmark.perf_args:
187      perf_args_list = self.benchmark.perf_args.split(' ')
188      perf_args_list = [perf_args_list[0]] + ['-a'] + perf_args_list[1:]
189      perf_args = ' '.join(perf_args_list)
190      if not perf_args_list[0] in ['record', 'stat']:
191        raise Exception('perf_args must start with either record or stat')
192      extra_test_args = ['--profiler=custom_perf',
193                         ("--profiler_args='perf_options=\"%s\"'" % perf_args)]
194      return ' '.join(extra_test_args)
195    else:
196      return ''
197
198  def RunTest(self, machine):
199    self.timeline.Record(STATUS_IMAGING)
200    if self.owner_thread is not None:
201      # In schedv2 mode, do not even call ImageMachine. Machine image is
202      # guarenteed.
203      pass
204    else:
205      self.machine_manager.ImageMachine(machine, self.label)
206    self.timeline.Record(STATUS_RUNNING)
207    [retval, out, err] = self.suite_runner.Run(machine.name, self.label,
208                                               self.benchmark, self.test_args,
209                                               self.profiler_args)
210    self.run_completed = True
211    return Result.CreateFromRun(self._logger, self.log_level, self.label,
212                                self.machine, out, err, retval,
213                                self.benchmark.show_all_results,
214                                self.benchmark.test_name, self.benchmark.suite)
215
216  def SetCacheConditions(self, cache_conditions):
217    self.cache_conditions = cache_conditions
218
219  def __str__(self):
220    """For better debugging."""
221
222    return 'BenchmarkRun[name="{}"]'.format(self.name)
223
224
225class MockBenchmarkRun(BenchmarkRun):
226  """Inherited from BenchmarkRun."""
227
228  def ReadCache(self):
229    # Just use the first machine for running the cached version,
230    # without locking it.
231    self.cache = MockResultsCache()
232    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
233                    self.benchmark.test_name, self.iteration, self.test_args,
234                    self.profiler_args, self.machine_manager, self.machine,
235                    self.label.board, self.cache_conditions, self._logger,
236                    self.log_level, self.label, self.share_cache,
237                    self.benchmark.suite, self.benchmark.show_all_results,
238                    self.benchmark.run_local)
239
240    self.result = self.cache.ReadResult()
241    self.cache_hit = (self.result is not None)
242
243  def RunTest(self, machine):
244    """Remove Result.CreateFromRun for testing."""
245    self.timeline.Record(STATUS_IMAGING)
246    self.machine_manager.ImageMachine(machine, self.label)
247    self.timeline.Record(STATUS_RUNNING)
248    [retval, out, err] = self.suite_runner.Run(machine.name, self.label,
249                                               self.benchmark, self.test_args,
250                                               self.profiler_args)
251    self.run_completed = True
252    rr = MockResult('logger', self.label, self.log_level, machine)
253    rr.out = out
254    rr.err = err
255    rr.retval = retval
256    return rr
257