experiment_factory.py revision 88272d479f2761cc1906fea564c73033f77a6270 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""A module to generate experiments."""
5
6from __future__ import print_function
7import os
8import re
9import socket
10
11from benchmark import Benchmark
12import config
13from experiment import Experiment
14from label import Label
15from label import MockLabel
16from results_cache import CacheConditions
17import test_flag
18import file_lock_machine
19
20# Users may want to run Telemetry tests either individually, or in
21# specified sets.  Here we define sets of tests that users may want
22# to run together.
23
24telemetry_perfv2_tests = ['dromaeo.domcoreattr',
25                          'dromaeo.domcoremodify',
26                          'dromaeo.domcorequery',
27                          'dromaeo.domcoretraverse',
28                          'kraken',
29                          'octane',
30                          'robohornet_pro',
31                          'sunspider']
32
33telemetry_pagecycler_tests = ['page_cycler.intl_ar_fa_he',
34                              'page_cycler.intl_es_fr_pt-BR',
35                              'page_cycler.intl_hi_ru',
36                              'page_cycler.intl_ja_zh',
37                              'page_cycler.intl_ko_th_vi',
38                              'page_cycler.morejs',
39                              'page_cycler.moz',
40                              'page_cycler.netsim.top_10',
41                              'page_cycler.tough_layout_cases',
42                              'page_cycler.typical_25']
43
44telemetry_toolchain_old_perf_tests = ['dromaeo.domcoremodify',
45                                      'page_cycler.intl_es_fr_pt-BR',
46                                      'page_cycler.intl_hi_ru',
47                                      'page_cycler.intl_ja_zh',
48                                      'page_cycler.intl_ko_th_vi',
49                                      'page_cycler.netsim.top_10',
50                                      'page_cycler.typical_25',
51                                      'robohornet_pro',
52                                      'spaceport',
53                                      'tab_switching.top_10']
54telemetry_toolchain_perf_tests = ['octane',
55                                  'kraken',
56                                  'speedometer',
57                                  'dromaeo.domcoreattr',
58                                  'dromaeo.domcoremodify',
59                                  'smoothness.tough_webgl_cases',
60                                  'page_cycler.typical_25',
61                                  'media.tough_video_cases']
62
63
64class ExperimentFactory(object):
65  """Factory class for building an Experiment, given an ExperimentFile as input.
66
67  This factory is currently hardcoded to produce an experiment for running
68  ChromeOS benchmarks, but the idea is that in the future, other types
69  of experiments could be produced.
70  """
71
72  def AppendBenchmarkSet(self, benchmarks, benchmark_list, test_args,
73                         iterations, rm_chroot_tmp, perf_args, suite,
74                         show_all_results, retries, run_local):
75    """Add all the tests in a set to the benchmarks list."""
76    for test_name in benchmark_list:
77      telemetry_benchmark = Benchmark(
78          test_name, test_name, test_args, iterations, rm_chroot_tmp, perf_args,
79          suite, show_all_results, retries, run_local)
80      benchmarks.append(telemetry_benchmark)
81
82  def GetExperiment(self, experiment_file, working_directory, log_dir):
83    """Construct an experiment from an experiment file."""
84    global_settings = experiment_file.GetGlobalSettings()
85    experiment_name = global_settings.GetField('name')
86    board = global_settings.GetField('board')
87    remote = global_settings.GetField('remote')
88    # This is used to remove the ",' from the remote if user
89    # add them to the remote string.
90    new_remote = []
91    if remote:
92      for i in remote:
93        c = re.sub('["\']', '', i)
94        new_remote.append(c)
95    remote = new_remote
96    chromeos_root = global_settings.GetField('chromeos_root')
97    rm_chroot_tmp = global_settings.GetField('rm_chroot_tmp')
98    perf_args = global_settings.GetField('perf_args')
99    acquire_timeout = global_settings.GetField('acquire_timeout')
100    cache_dir = global_settings.GetField('cache_dir')
101    cache_only = global_settings.GetField('cache_only')
102    config.AddConfig('no_email', global_settings.GetField('no_email'))
103    share_cache = global_settings.GetField('share_cache')
104    results_dir = global_settings.GetField('results_dir')
105    use_file_locks = global_settings.GetField('use_file_locks')
106    locks_dir = global_settings.GetField('locks_dir')
107    # If we pass a blank locks_dir to the Experiment, it will use the AFE server
108    # lock mechanism.  So if the user specified use_file_locks, but did not
109    # specify a locks dir, set the locks  dir to the default locks dir in
110    # file_lock_machine.
111    if use_file_locks and not locks_dir:
112      locks_dir = file_lock_machine.Machine.LOCKS_DIR
113    chrome_src = global_settings.GetField('chrome_src')
114    show_all_results = global_settings.GetField('show_all_results')
115    log_level = global_settings.GetField('logging_level')
116    if log_level not in ('quiet', 'average', 'verbose'):
117      log_level = 'verbose'
118    # Default cache hit conditions. The image checksum in the cache and the
119    # computed checksum of the image must match. Also a cache file must exist.
120    cache_conditions = [CacheConditions.CACHE_FILE_EXISTS,
121                        CacheConditions.CHECKSUMS_MATCH]
122    if global_settings.GetField('rerun_if_failed'):
123      cache_conditions.append(CacheConditions.RUN_SUCCEEDED)
124    if global_settings.GetField('rerun'):
125      cache_conditions.append(CacheConditions.FALSE)
126    if global_settings.GetField('same_machine'):
127      cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH)
128    if global_settings.GetField('same_specs'):
129      cache_conditions.append(CacheConditions.MACHINES_MATCH)
130
131    # Construct benchmarks.
132    # Some fields are common with global settings. The values are
133    # inherited and/or merged with the global settings values.
134    benchmarks = []
135    all_benchmark_settings = experiment_file.GetSettings('benchmark')
136    for benchmark_settings in all_benchmark_settings:
137      benchmark_name = benchmark_settings.name
138      test_name = benchmark_settings.GetField('test_name')
139      if not test_name:
140        test_name = benchmark_name
141      test_args = benchmark_settings.GetField('test_args')
142      iterations = benchmark_settings.GetField('iterations')
143      suite = benchmark_settings.GetField('suite')
144      retries = benchmark_settings.GetField('retries')
145      run_local = benchmark_settings.GetField('run_local')
146
147      if suite == 'telemetry_Crosperf':
148        if test_name == 'all_perfv2':
149          self.AppendBenchmarkSet(benchmarks, telemetry_perfv2_tests,
150                                  test_args, iterations, rm_chroot_tmp,
151                                  perf_args, suite, show_all_results, retries,
152                                  run_local)
153        elif test_name == 'all_pagecyclers':
154          self.AppendBenchmarkSet(benchmarks, telemetry_pagecycler_tests,
155                                  test_args, iterations, rm_chroot_tmp,
156                                  perf_args, suite, show_all_results, retries,
157                                  run_local)
158        elif test_name == 'all_toolchain_perf':
159          self.AppendBenchmarkSet(benchmarks, telemetry_toolchain_perf_tests,
160                                  test_args, iterations, rm_chroot_tmp,
161                                  perf_args, suite, show_all_results, retries,
162                                  run_local)
163          # Add non-telemetry toolchain-perf benchmarks:
164          benchmarks.append(Benchmark('graphics_WebGLAquarium',
165                                      'graphics_WebGLAquarium',
166                                      '',
167                                      iterations,
168                                      rm_chroot_tmp,
169                                      perf_args,
170                                      '',
171                                      show_all_results,
172                                      retries,
173                                      run_local=False))
174        elif test_name == 'all_toolchain_perf_old':
175          self.AppendBenchmarkSet(
176              benchmarks, telemetry_toolchain_old_perf_tests, test_args,
177              iterations, rm_chroot_tmp, perf_args, suite, show_all_results,
178              retries, run_local)
179        else:
180          benchmark = Benchmark(test_name, test_name, test_args, iterations,
181                                rm_chroot_tmp, perf_args, suite,
182                                show_all_results, retries, run_local)
183          benchmarks.append(benchmark)
184      else:
185        # Add the single benchmark.
186        benchmark = Benchmark(benchmark_name,
187                              test_name,
188                              test_args,
189                              iterations,
190                              rm_chroot_tmp,
191                              perf_args,
192                              suite,
193                              show_all_results,
194                              retries,
195                              run_local=False)
196        benchmarks.append(benchmark)
197
198    if not benchmarks:
199      raise RuntimeError('No benchmarks specified')
200
201    # Construct labels.
202    # Some fields are common with global settings. The values are
203    # inherited and/or merged with the global settings values.
204    labels = []
205    all_label_settings = experiment_file.GetSettings('label')
206    all_remote = list(remote)
207    for label_settings in all_label_settings:
208      label_name = label_settings.name
209      image = label_settings.GetField('chromeos_image')
210      chromeos_root = label_settings.GetField('chromeos_root')
211      my_remote = label_settings.GetField('remote')
212      compiler = label_settings.GetField('compiler')
213      new_remote = []
214      if my_remote:
215        for i in my_remote:
216          c = re.sub('["\']', '', i)
217          new_remote.append(c)
218      my_remote = new_remote
219      if image == '':
220        build = label_settings.GetField('build')
221        if len(build) == 0:
222          raise RuntimeError("Can not have empty 'build' field!")
223        image = label_settings.GetXbuddyPath(build, board, chromeos_root,
224                                             log_level)
225
226      cache_dir = label_settings.GetField('cache_dir')
227      chrome_src = label_settings.GetField('chrome_src')
228
229      # TODO(yunlian): We should consolidate code in machine_manager.py
230      # to derermine whether we are running from within google or not
231      if ('corp.google.com' in socket.gethostname() and
232          (not my_remote or my_remote == remote and
233           global_settings.GetField('board') != board)):
234        my_remote = self.GetDefaultRemotes(board)
235      if global_settings.GetField('same_machine') and len(my_remote) > 1:
236        raise RuntimeError('Only one remote is allowed when same_machine '
237                           'is turned on')
238      all_remote += my_remote
239      image_args = label_settings.GetField('image_args')
240      if test_flag.GetTestMode():
241        # pylint: disable=too-many-function-args
242        label = MockLabel(label_name, image, chromeos_root, board, my_remote,
243                          image_args, cache_dir, cache_only, log_level,
244                          compiler, chrome_src)
245      else:
246        label = Label(label_name, image, chromeos_root, board, my_remote,
247                      image_args, cache_dir, cache_only, log_level, compiler,
248                      chrome_src)
249      labels.append(label)
250
251    if not labels:
252      raise RuntimeError('No labels specified')
253
254    email = global_settings.GetField('email')
255    all_remote += list(set(my_remote))
256    all_remote = list(set(all_remote))
257    experiment = Experiment(experiment_name, all_remote, working_directory,
258                            chromeos_root, cache_conditions, labels, benchmarks,
259                            experiment_file.Canonicalize(), email,
260                            acquire_timeout, log_dir, log_level, share_cache,
261                            results_dir, locks_dir)
262
263    return experiment
264
265  def GetDefaultRemotes(self, board):
266    default_remotes_file = os.path.join(
267        os.path.dirname(__file__), 'default_remotes')
268    try:
269      with open(default_remotes_file) as f:
270        for line in f:
271          key, v = line.split(':')
272          if key.strip() == board:
273            remotes = v.strip().split(' ')
274            if remotes:
275              return remotes
276            else:
277              raise RuntimeError('There is no remote for {0}'.format(board))
278    except IOError:
279      # TODO: rethrow instead of throwing different exception.
280      raise RuntimeError('IOError while reading file {0}'
281                         .format(default_remotes_file))
282    else:
283      raise RuntimeError('There is not remote for {0}'.format(board))
284