experiment_factory_unittest.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import StringIO
8import socket
9import mock
10import unittest
11
12from cros_utils.file_utils import FileUtils
13
14from experiment_factory import ExperimentFactory
15from experiment_file import ExperimentFile
16import test_flag
17import benchmark
18import label
19import experiment
20import experiment_factory
21import machine_manager
22import settings_factory
23import test_flag
24
25EXPERIMENT_FILE_1 = """
26  board: x86-alex
27  remote: chromeos-alex3
28
29  benchmark: PageCycler {
30    iterations: 3
31  }
32
33  image1 {
34    chromeos_image: /usr/local/google/cros_image1.bin
35  }
36
37  image2 {
38    chromeos_image: /usr/local/google/cros_image2.bin
39  }
40  """
41
42
43class ExperimentFactoryTest(unittest.TestCase):
44
45  def testLoadExperimentFile1(self):
46    experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1))
47    experiment = ExperimentFactory().GetExperiment(experiment_file,
48                                                   working_directory='',
49                                                   log_dir='')
50    self.assertEqual(experiment.remote, ['chromeos-alex3'])
51
52    self.assertEqual(len(experiment.benchmarks), 1)
53    self.assertEqual(experiment.benchmarks[0].name, 'PageCycler')
54    self.assertEqual(experiment.benchmarks[0].test_name, 'PageCycler')
55    self.assertEqual(experiment.benchmarks[0].iterations, 3)
56
57    self.assertEqual(len(experiment.labels), 2)
58    self.assertEqual(experiment.labels[0].chromeos_image,
59                     '/usr/local/google/cros_image1.bin')
60    self.assertEqual(experiment.labels[0].board, 'x86-alex')
61
62  def test_append_benchmark_set(self):
63    ef = ExperimentFactory()
64
65    bench_list = []
66    ef._AppendBenchmarkSet(bench_list,
67                           experiment_factory.telemetry_perfv2_tests, '', 1,
68                           False, '', 'telemetry_Crosperf', False, 0, False)
69    self.assertEqual(
70        len(bench_list), len(experiment_factory.telemetry_perfv2_tests))
71    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
72
73    bench_list = []
74    ef._AppendBenchmarkSet(bench_list,
75                           experiment_factory.telemetry_pagecycler_tests, '', 1,
76                           False, '', 'telemetry_Crosperf', False, 0, False)
77    self.assertEqual(
78        len(bench_list), len(experiment_factory.telemetry_pagecycler_tests))
79    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
80
81    bench_list = []
82    ef._AppendBenchmarkSet(
83        bench_list, experiment_factory.telemetry_toolchain_perf_tests, '', 1,
84        False, '', 'telemetry_Crosperf', False, 0, False)
85    self.assertEqual(
86        len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests))
87    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
88
89  @mock.patch.object(socket, 'gethostname')
90  @mock.patch.object(machine_manager.MachineManager, 'AddMachine')
91  def test_get_experiment(self, mock_machine_manager, mock_socket):
92
93    test_flag.SetTestMode(False)
94    self.append_benchmark_call_args = []
95
96    def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch,
97                               perf_args, suite, show_all):
98      'Helper function for test_get_experiment'
99      arg_list = [bench_list, set_list, args, iters, rm_ch, perf_args, suite,
100                  show_all]
101      self.append_benchmark_call_args.append(args_list)
102
103    def FakeGetDefaultRemotes(board):
104      return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']
105
106    def FakeGetXbuddyPath(build, board, chroot, log_level):
107      return 'fake_image_path'
108
109    ef = ExperimentFactory()
110    ef._AppendBenchmarkSet = FakeAppendBenchmarkSet
111    ef.GetDefaultRemotes = FakeGetDefaultRemotes
112
113    label_settings = settings_factory.LabelSettings('image_label')
114    benchmark_settings = settings_factory.BenchmarkSettings('bench_test')
115    global_settings = settings_factory.GlobalSettings('test_name')
116
117    label_settings.GetXbuddyPath = FakeGetXbuddyPath
118
119    mock_experiment_file = ExperimentFile(StringIO.StringIO(''))
120    mock_experiment_file.all_settings = []
121
122    test_flag.SetTestMode(True)
123    # Basic test.
124    global_settings.SetField('name', 'unittest_test')
125    global_settings.SetField('board', 'lumpy')
126    global_settings.SetField('remote', '123.45.67.89 123.45.76.80')
127    benchmark_settings.SetField('test_name', 'kraken')
128    benchmark_settings.SetField('suite', 'telemetry_Crosperf')
129    benchmark_settings.SetField('iterations', 1)
130    label_settings.SetField(
131        'chromeos_image',
132        'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin')
133    label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top')
134
135    mock_experiment_file.global_settings = global_settings
136    mock_experiment_file.all_settings.append(label_settings)
137    mock_experiment_file.all_settings.append(benchmark_settings)
138    mock_experiment_file.all_settings.append(global_settings)
139
140    mock_socket.return_value = ''
141
142    # First test. General test.
143    exp = ef.GetExperiment(mock_experiment_file, '', '')
144    self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
145    self.assertEqual(exp.cache_conditions, [0, 2, 1])
146    self.assertEqual(exp.log_level, 'average')
147
148    self.assertEqual(len(exp.benchmarks), 1)
149    self.assertEqual(exp.benchmarks[0].name, 'kraken')
150    self.assertEqual(exp.benchmarks[0].test_name, 'kraken')
151    self.assertEqual(exp.benchmarks[0].iterations, 1)
152    self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf')
153    self.assertFalse(exp.benchmarks[0].show_all_results)
154
155    self.assertEqual(len(exp.labels), 1)
156    self.assertEqual(exp.labels[0].chromeos_image,
157                     'chromeos/src/build/images/lumpy/latest/'
158                     'chromiumos_test_image.bin')
159    self.assertEqual(exp.labels[0].board, 'lumpy')
160
161    # Second test: Remotes listed in labels.
162    test_flag.SetTestMode(True)
163    label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros')
164    exp = ef.GetExperiment(mock_experiment_file, '', '')
165    self.assertEqual(exp.remote, ['chromeos1.cros',
166                                  'chromeos2.cros',
167                                  '123.45.67.89',
168                                  '123.45.76.80'])
169
170    # Third test: Automatic fixing of bad  logging_level param:
171    global_settings.SetField('logging_level', 'really loud!')
172    exp = ef.GetExperiment(mock_experiment_file, '', '')
173    self.assertEqual(exp.log_level, 'verbose')
174
175    # Fourth test: Setting cache conditions; only 1 remote with "same_machine"
176    global_settings.SetField('rerun_if_failed', 'true')
177    global_settings.SetField('rerun', 'true')
178    global_settings.SetField('same_machine', 'true')
179    global_settings.SetField('same_specs', 'true')
180
181    self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '')
182    label_settings.SetField('remote', '')
183    global_settings.SetField('remote', '123.45.67.89')
184    exp = ef.GetExperiment(mock_experiment_file, '', '')
185    self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1])
186
187    # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all
188    # remotes (Call GetDefaultRemotes).
189    mock_socket.return_value = 'test.corp.google.com'
190    global_settings.SetField('remote', '')
191    global_settings.SetField('same_machine', 'false')
192
193    label_settings_2 = settings_factory.LabelSettings('official_image_label')
194    label_settings_2.SetField('chromeos_root', 'chromeos')
195    label_settings_2.SetField('build', 'official-dev')
196    label_settings_2.GetXbuddyPath = FakeGetXbuddyPath
197
198    mock_experiment_file.all_settings.append(label_settings_2)
199    exp = ef.GetExperiment(mock_experiment_file, '', '')
200    self.assertEqual(len(exp.labels), 2)
201    self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path')
202    self.assertEqual(exp.remote, ['fake_chromeos_machine1.cros',
203                                  'fake_chromeos_machine2.cros'])
204
205  def test_get_default_remotes(self):
206    board_list = ['x86-alex', 'lumpy', 'stumpy', 'parrot', 'daisy', 'peach_pit',
207                  'peppy', 'squawks']
208
209    ef = ExperimentFactory()
210    self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board')
211
212    # Verify that we have entries for every board, and that we get three
213    # machines back for each board.
214    for b in board_list:
215      remotes = ef.GetDefaultRemotes(b)
216      self.assertEqual(len(remotes), 3)
217
218
219if __name__ == '__main__':
220  FileUtils.Configure(True)
221  test_flag.SetTestMode(True)
222  unittest.main()
223