binary_search_tool_tester.py revision b1d0c4e00ed787befa9591231ac5d04d2cf6795d
1#!/usr/bin/python2 2 3# Copyright 2012 Google Inc. All Rights Reserved. 4"""Tests for bisecting tool.""" 5 6from __future__ import print_function 7 8__author__ = 'shenhan@google.com (Han Shen)' 9 10import os 11import random 12import sys 13import unittest 14 15from cros_utils import command_executer 16from binary_search_tool import binary_search_state 17from binary_search_tool import bisect 18 19import common 20import gen_obj 21 22 23def GenObj(): 24 obj_num = random.randint(100, 1000) 25 bad_obj_num = random.randint(obj_num / 100, obj_num / 20) 26 if bad_obj_num == 0: 27 bad_obj_num = 1 28 gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)]) 29 30 31def CleanObj(): 32 os.remove(common.OBJECTS_FILE) 33 os.remove(common.WORKING_SET_FILE) 34 print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE, 35 common.WORKING_SET_FILE)) 36 37 38class BisectTest(unittest.TestCase): 39 """Tests for bisect.py""" 40 41 def setUp(self): 42 with open('./installed', 'w'): 43 pass 44 45 try: 46 os.remove(binary_search_state.STATE_FILE) 47 except OSError: 48 pass 49 50 def tearDown(self): 51 try: 52 os.remove('./installed') 53 os.remove(os.readlink(binary_search_state.STATE_FILE)) 54 os.remove(binary_search_state.STATE_FILE) 55 except OSError: 56 pass 57 58 class FullBisector(bisect.Bisector): 59 """Test bisector to test bisect.py with""" 60 61 def __init__(self, options, overrides): 62 super(BisectTest.FullBisector, self).__init__(options, overrides) 63 64 def PreRun(self): 65 GenObj() 66 return 0 67 68 def Run(self): 69 return binary_search_state.Run(get_initial_items='./gen_init_list.py', 70 switch_to_good='./switch_to_good.py', 71 switch_to_bad='./switch_to_bad.py', 72 test_script='./is_good.py', 73 prune=True, 74 file_args=True) 75 76 def PostRun(self): 77 CleanObj() 78 return 0 79 80 def test_full_bisector(self): 81 ret = bisect.Run(self.FullBisector({}, {})) 82 self.assertEquals(ret, 0) 83 self.assertFalse(os.path.exists(common.OBJECTS_FILE)) 84 self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) 85 86 def check_output(self): 87 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 88 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 89 'tail -n1')) 90 ls = out.splitlines() 91 self.assertEqual(len(ls), 1) 92 line = ls[0] 93 94 _, _, bad_ones = line.partition('Bad items are: ') 95 bad_ones = bad_ones.split() 96 expected_result = common.ReadObjectsFile() 97 98 # Reconstruct objects file from bad_ones and compare 99 actual_result = [0] * len(expected_result) 100 for bad_obj in bad_ones: 101 actual_result[int(bad_obj)] = 1 102 103 self.assertEqual(actual_result, expected_result) 104 105 106class BisectingUtilsTest(unittest.TestCase): 107 """Tests for bisecting tool.""" 108 109 def setUp(self): 110 """Generate [100-1000] object files, and 1-5% of which are bad ones.""" 111 GenObj() 112 113 with open('./installed', 'w'): 114 pass 115 116 try: 117 os.remove(binary_search_state.STATE_FILE) 118 except OSError: 119 pass 120 121 def tearDown(self): 122 """Cleanup temp files.""" 123 CleanObj() 124 125 try: 126 os.remove(os.readlink(binary_search_state.STATE_FILE)) 127 except OSError: 128 pass 129 130 cleanup_list = ['./installed', 131 binary_search_state.STATE_FILE, 132 'noinc_prune_bad', 133 'noinc_prune_good'] 134 for f in cleanup_list: 135 if os.path.exists(f): 136 os.remove(f) 137 138 def runTest(self): 139 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 140 switch_to_good='./switch_to_good.py', 141 switch_to_bad='./switch_to_bad.py', 142 test_script='./is_good.py', 143 prune=True, 144 file_args=True) 145 self.assertEquals(ret, 0) 146 self.check_output() 147 148 def test_arg_parse(self): 149 args = ['--get_initial_items', './gen_init_list.py', '--switch_to_good', 150 './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py', 151 '--test_script', './is_good.py', '--prune', '--file_args'] 152 ret = binary_search_state.Main(args) 153 self.assertEquals(ret, 0) 154 self.check_output() 155 156 def test_install_script(self): 157 os.remove('./installed') 158 with self.assertRaises(AssertionError): 159 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 160 switch_to_good='./switch_to_good.py', 161 switch_to_bad='./switch_to_bad.py', 162 test_script='./is_good.py', 163 prune=True, 164 file_args=True) 165 166 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 167 switch_to_good='./switch_to_good.py', 168 switch_to_bad='./switch_to_bad.py', 169 test_script='./is_good.py', 170 install_script='./install.py', 171 prune=True, 172 file_args=True) 173 self.assertEquals(ret, 0) 174 self.check_output() 175 176 def test_bad_install_script(self): 177 with self.assertRaises(AssertionError): 178 binary_search_state.Run(get_initial_items='./gen_init_list.py', 179 switch_to_good='./switch_to_good.py', 180 switch_to_bad='./switch_to_bad.py', 181 test_script='./is_good.py', 182 install_script='./install_bad.py', 183 prune=True, 184 file_args=True) 185 186 def test_bad_save_state(self): 187 state_file = binary_search_state.STATE_FILE 188 hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE) 189 190 with open(state_file, 'w') as f: 191 f.write('test123') 192 193 bss = binary_search_state.MockBinarySearchState() 194 with self.assertRaises(binary_search_state.Error): 195 bss.SaveState() 196 197 with open(state_file, 'r') as f: 198 self.assertEquals(f.read(), 'test123') 199 200 os.remove(state_file) 201 202 # Cleanup generated save state that has no symlink 203 files = os.listdir(os.getcwd()) 204 save_states = [x for x in files if x.startswith(hidden_state_file)] 205 _ = [os.remove(x) for x in save_states] 206 207 def test_save_state(self): 208 state_file = binary_search_state.STATE_FILE 209 210 bss = binary_search_state.MockBinarySearchState() 211 bss.SaveState() 212 self.assertTrue(os.path.exists(state_file)) 213 first_state = os.readlink(state_file) 214 215 bss.SaveState() 216 second_state = os.readlink(state_file) 217 self.assertTrue(os.path.exists(state_file)) 218 self.assertTrue(second_state != first_state) 219 self.assertFalse(os.path.exists(first_state)) 220 221 bss.RemoveState() 222 self.assertFalse(os.path.islink(state_file)) 223 self.assertFalse(os.path.exists(second_state)) 224 225 def test_load_state(self): 226 test_items = [1, 2, 3, 4, 5] 227 228 bss = binary_search_state.MockBinarySearchState() 229 bss.all_items = test_items 230 bss.SaveState() 231 232 bss = None 233 234 bss2 = binary_search_state.MockBinarySearchState.LoadState() 235 self.assertEquals(bss2.all_items, test_items) 236 237 def test_tmp_cleanup(self): 238 bss = binary_search_state.MockBinarySearchState( 239 get_initial_items='echo "0\n1\n2\n3"', 240 switch_to_good='./switch_tmp.py', 241 file_args=True) 242 bss.SwitchToGood(['0', '1', '2', '3']) 243 244 tmp_file = None 245 with open('tmp_file', 'r') as f: 246 tmp_file = f.read() 247 os.remove('tmp_file') 248 249 self.assertFalse(os.path.exists(tmp_file)) 250 ws = common.ReadWorkingSet() 251 for i in range(3): 252 self.assertEquals(ws[i], 42) 253 254 def test_verify_fail(self): 255 bss = binary_search_state.MockBinarySearchState( 256 get_initial_items='./gen_init_list.py', 257 switch_to_good='./switch_to_bad.py', 258 switch_to_bad='./switch_to_good.py', 259 test_script='./is_good.py', 260 prune=True, 261 file_args=True, 262 verify=True) 263 with self.assertRaises(AssertionError): 264 bss.DoVerify() 265 266 def test_early_terminate(self): 267 bss = binary_search_state.MockBinarySearchState( 268 get_initial_items='./gen_init_list.py', 269 switch_to_good='./switch_to_good.py', 270 switch_to_bad='./switch_to_bad.py', 271 test_script='./is_good.py', 272 prune=True, 273 file_args=True, 274 iterations=1) 275 bss.DoSearch() 276 self.assertFalse(bss.found_items) 277 278 def test_no_prune(self): 279 bss = binary_search_state.MockBinarySearchState( 280 get_initial_items='./gen_init_list.py', 281 switch_to_good='./switch_to_good.py', 282 switch_to_bad='./switch_to_bad.py', 283 test_script='./is_good.py', 284 install_script='./install.py', 285 prune=False, 286 file_args=True) 287 bss.DoSearch() 288 self.assertEquals(len(bss.found_items), 1) 289 290 bad_objs = common.ReadObjectsFile() 291 found_obj = int(bss.found_items.pop()) 292 self.assertEquals(bad_objs[found_obj], 1) 293 294 def test_set_file(self): 295 binary_search_state.Run( 296 get_initial_items='./gen_init_list.py', 297 switch_to_good='./switch_to_good_set_file.py', 298 switch_to_bad='./switch_to_bad_set_file.py', 299 test_script='./is_good.py', 300 prune=True, 301 file_args=True, 302 verify=True) 303 self.check_output() 304 305 def test_noincremental_prune(self): 306 ret = binary_search_state.Run( 307 get_initial_items='./gen_init_list.py', 308 switch_to_good='./switch_to_good_noinc_prune.py', 309 switch_to_bad='./switch_to_bad_noinc_prune.py', 310 test_script='./is_good_noinc_prune.py', 311 install_script='./install.py', 312 prune=True, 313 noincremental=True, 314 file_args=True, 315 verify=False) 316 self.assertEquals(ret, 0) 317 self.check_output() 318 319 def check_output(self): 320 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 321 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 322 'tail -n1')) 323 ls = out.splitlines() 324 self.assertEqual(len(ls), 1) 325 line = ls[0] 326 327 _, _, bad_ones = line.partition('Bad items are: ') 328 bad_ones = bad_ones.split() 329 expected_result = common.ReadObjectsFile() 330 331 # Reconstruct objects file from bad_ones and compare 332 actual_result = [0] * len(expected_result) 333 for bad_obj in bad_ones: 334 actual_result[int(bad_obj)] = 1 335 336 self.assertEqual(actual_result, expected_result) 337 338 339class BisectStressTest(unittest.TestCase): 340 """Stress tests for bisecting tool.""" 341 342 def test_every_obj_bad(self): 343 amt = 25 344 gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)]) 345 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 346 switch_to_good='./switch_to_good.py', 347 switch_to_bad='./switch_to_bad.py', 348 test_script='./is_good.py', 349 prune=True, 350 file_args=True, 351 verify=False) 352 self.assertEquals(ret, 0) 353 self.check_output() 354 355 def test_every_index_is_bad(self): 356 amt = 25 357 for i in range(amt): 358 obj_list = ['0'] * amt 359 obj_list[i] = '1' 360 obj_list = ','.join(obj_list) 361 gen_obj.Main(['--obj_list', obj_list]) 362 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 363 switch_to_good='./switch_to_good.py', 364 switch_to_bad='./switch_to_bad.py', 365 install_script='./install.py', 366 test_script='./is_good.py', 367 prune=True, 368 file_args=True) 369 self.assertEquals(ret, 0) 370 self.check_output() 371 372 373 def check_output(self): 374 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 375 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 376 'tail -n1')) 377 ls = out.splitlines() 378 self.assertEqual(len(ls), 1) 379 line = ls[0] 380 381 _, _, bad_ones = line.partition('Bad items are: ') 382 bad_ones = bad_ones.split() 383 expected_result = common.ReadObjectsFile() 384 385 # Reconstruct objects file from bad_ones and compare 386 actual_result = [0] * len(expected_result) 387 for bad_obj in bad_ones: 388 actual_result[int(bad_obj)] = 1 389 390 self.assertEqual(actual_result, expected_result) 391 392 393def Main(argv): 394 num_tests = 2 395 if len(argv) > 1: 396 num_tests = int(argv[1]) 397 398 suite = unittest.TestSuite() 399 for _ in range(0, num_tests): 400 suite.addTest(BisectingUtilsTest()) 401 suite.addTest(BisectingUtilsTest('test_arg_parse')) 402 suite.addTest(BisectingUtilsTest('test_install_script')) 403 suite.addTest(BisectingUtilsTest('test_bad_install_script')) 404 suite.addTest(BisectingUtilsTest('test_bad_save_state')) 405 suite.addTest(BisectingUtilsTest('test_save_state')) 406 suite.addTest(BisectingUtilsTest('test_load_state')) 407 suite.addTest(BisectingUtilsTest('test_tmp_cleanup')) 408 suite.addTest(BisectingUtilsTest('test_verify_fail')) 409 suite.addTest(BisectingUtilsTest('test_early_terminate')) 410 suite.addTest(BisectingUtilsTest('test_no_prune')) 411 suite.addTest(BisectingUtilsTest('test_set_file')) 412 suite.addTest(BisectingUtilsTest('test_noincremental_prune')) 413 suite.addTest(BisectTest('test_full_bisector')) 414 suite.addTest(BisectStressTest('test_every_obj_bad')) 415 suite.addTest(BisectStressTest('test_every_index_is_bad')) 416 runner = unittest.TextTestRunner() 417 runner.run(suite) 418 419 420if __name__ == '__main__': 421 Main(sys.argv) 422