experiment_factory_unittest.py revision 88272d479f2761cc1906fea564c73033f77a6270
1#!/usr/bin/python2 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Unit test for experiment_factory.py""" 7 8from __future__ import print_function 9 10import StringIO 11import socket 12import mock 13import unittest 14 15from cros_utils.file_utils import FileUtils 16 17from experiment_factory import ExperimentFactory 18from experiment_file import ExperimentFile 19import test_flag 20import benchmark 21import experiment_factory 22import settings_factory 23 24EXPERIMENT_FILE_1 = """ 25 board: x86-alex 26 remote: chromeos-alex3 27 28 benchmark: PageCycler { 29 iterations: 3 30 } 31 32 image1 { 33 chromeos_image: /usr/local/google/cros_image1.bin 34 } 35 36 image2 { 37 chromeos_image: /usr/local/google/cros_image2.bin 38 } 39 """ 40 41# pylint: disable=too-many-function-args 42 43class ExperimentFactoryTest(unittest.TestCase): 44 """Class for running experiment factory unittests.""" 45 46 def setUp(self): 47 self.append_benchmark_call_args = [] 48 49 def testLoadExperimentFile1(self): 50 experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) 51 exp = ExperimentFactory().GetExperiment(experiment_file, 52 working_directory='', 53 log_dir='') 54 self.assertEqual(exp.remote, ['chromeos-alex3']) 55 56 self.assertEqual(len(exp.benchmarks), 1) 57 self.assertEqual(exp.benchmarks[0].name, 'PageCycler') 58 self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler') 59 self.assertEqual(exp.benchmarks[0].iterations, 3) 60 61 self.assertEqual(len(exp.labels), 2) 62 self.assertEqual(exp.labels[0].chromeos_image, 63 '/usr/local/google/cros_image1.bin') 64 self.assertEqual(exp.labels[0].board, 'x86-alex') 65 66 def test_append_benchmark_set(self): 67 ef = ExperimentFactory() 68 69 bench_list = [] 70 ef.AppendBenchmarkSet(bench_list, 71 experiment_factory.telemetry_perfv2_tests, '', 1, 72 False, '', 'telemetry_Crosperf', False, 73 0, 74 False) 75 self.assertEqual( 76 len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) 77 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 78 79 bench_list = [] 80 ef.AppendBenchmarkSet(bench_list, 81 experiment_factory.telemetry_pagecycler_tests, '', 1, 82 False, '', 'telemetry_Crosperf', False, 0, False) 83 self.assertEqual( 84 len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) 85 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 86 87 bench_list = [] 88 ef.AppendBenchmarkSet( 89 bench_list, experiment_factory.telemetry_toolchain_perf_tests, '', 1, 90 False, '', 'telemetry_Crosperf', False, 0, False) 91 self.assertEqual( 92 len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) 93 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 94 95 @mock.patch.object(socket, 'gethostname') 96 def test_get_experiment(self, mock_socket): 97 98 test_flag.SetTestMode(False) 99 self.append_benchmark_call_args = [] 100 101 def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch, 102 perf_args, suite, show_all): 103 'Helper function for test_get_experiment' 104 arg_list = [bench_list, set_list, args, iters, rm_ch, perf_args, suite, 105 show_all] 106 self.append_benchmark_call_args.append(arg_list) 107 108 def FakeGetDefaultRemotes(board): 109 if not board: 110 return [] 111 return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'] 112 113 def FakeGetXbuddyPath(build, board, chroot, log_level): 114 if not build or not board or not chroot or not log_level: 115 return '' 116 return 'fake_image_path' 117 118 ef = ExperimentFactory() 119 ef.AppendBenchmarkSet = FakeAppendBenchmarkSet 120 ef.GetDefaultRemotes = FakeGetDefaultRemotes 121 122 label_settings = settings_factory.LabelSettings('image_label') 123 benchmark_settings = settings_factory.BenchmarkSettings('bench_test') 124 global_settings = settings_factory.GlobalSettings('test_name') 125 126 label_settings.GetXbuddyPath = FakeGetXbuddyPath 127 128 mock_experiment_file = ExperimentFile(StringIO.StringIO('')) 129 mock_experiment_file.all_settings = [] 130 131 test_flag.SetTestMode(True) 132 # Basic test. 133 global_settings.SetField('name', 'unittest_test') 134 global_settings.SetField('board', 'lumpy') 135 global_settings.SetField('remote', '123.45.67.89 123.45.76.80') 136 benchmark_settings.SetField('test_name', 'kraken') 137 benchmark_settings.SetField('suite', 'telemetry_Crosperf') 138 benchmark_settings.SetField('iterations', 1) 139 label_settings.SetField( 140 'chromeos_image', 141 'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin') 142 label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top') 143 144 mock_experiment_file.global_settings = global_settings 145 mock_experiment_file.all_settings.append(label_settings) 146 mock_experiment_file.all_settings.append(benchmark_settings) 147 mock_experiment_file.all_settings.append(global_settings) 148 149 mock_socket.return_value = '' 150 151 # First test. General test. 152 exp = ef.GetExperiment(mock_experiment_file, '', '') 153 self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) 154 self.assertEqual(exp.cache_conditions, [0, 2, 1]) 155 self.assertEqual(exp.log_level, 'average') 156 157 self.assertEqual(len(exp.benchmarks), 1) 158 self.assertEqual(exp.benchmarks[0].name, 'kraken') 159 self.assertEqual(exp.benchmarks[0].test_name, 'kraken') 160 self.assertEqual(exp.benchmarks[0].iterations, 1) 161 self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf') 162 self.assertFalse(exp.benchmarks[0].show_all_results) 163 164 self.assertEqual(len(exp.labels), 1) 165 self.assertEqual(exp.labels[0].chromeos_image, 166 'chromeos/src/build/images/lumpy/latest/' 167 'chromiumos_test_image.bin') 168 self.assertEqual(exp.labels[0].board, 'lumpy') 169 170 # Second test: Remotes listed in labels. 171 test_flag.SetTestMode(True) 172 label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros') 173 exp = ef.GetExperiment(mock_experiment_file, '', '') 174 self.assertEqual(exp.remote, ['chromeos1.cros', 175 'chromeos2.cros', 176 '123.45.67.89', 177 '123.45.76.80']) 178 179 # Third test: Automatic fixing of bad logging_level param: 180 global_settings.SetField('logging_level', 'really loud!') 181 exp = ef.GetExperiment(mock_experiment_file, '', '') 182 self.assertEqual(exp.log_level, 'verbose') 183 184 # Fourth test: Setting cache conditions; only 1 remote with "same_machine" 185 global_settings.SetField('rerun_if_failed', 'true') 186 global_settings.SetField('rerun', 'true') 187 global_settings.SetField('same_machine', 'true') 188 global_settings.SetField('same_specs', 'true') 189 190 self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') 191 label_settings.SetField('remote', '') 192 global_settings.SetField('remote', '123.45.67.89') 193 exp = ef.GetExperiment(mock_experiment_file, '', '') 194 self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) 195 196 # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all 197 # remotes (Call GetDefaultRemotes). 198 mock_socket.return_value = 'test.corp.google.com' 199 global_settings.SetField('remote', '') 200 global_settings.SetField('same_machine', 'false') 201 202 label_settings_2 = settings_factory.LabelSettings('official_image_label') 203 label_settings_2.SetField('chromeos_root', 'chromeos') 204 label_settings_2.SetField('build', 'official-dev') 205 label_settings_2.GetXbuddyPath = FakeGetXbuddyPath 206 207 mock_experiment_file.all_settings.append(label_settings_2) 208 exp = ef.GetExperiment(mock_experiment_file, '', '') 209 self.assertEqual(len(exp.labels), 2) 210 self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') 211 self.assertEqual(exp.remote, ['fake_chromeos_machine1.cros', 212 'fake_chromeos_machine2.cros']) 213 214 def test_get_default_remotes(self): 215 board_list = ['x86-alex', 'lumpy', 'stumpy', 'parrot', 'daisy', 'peach_pit', 216 'peppy', 'squawks'] 217 218 ef = ExperimentFactory() 219 self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board') 220 221 # Verify that we have entries for every board, and that we get three 222 # machines back for each board. 223 for b in board_list: 224 remotes = ef.GetDefaultRemotes(b) 225 self.assertEqual(len(remotes), 3) 226 227 228if __name__ == '__main__': 229 FileUtils.Configure(True) 230 test_flag.SetTestMode(True) 231 unittest.main() 232