binary_search_tool_tester.py revision a19ea38e4482af3bc98365be773305b5167eddfd 
1#!/usr/bin/python2
2
3# Copyright 2012 Google Inc. All Rights Reserved.
4"""Tests for bisecting tool."""
5
6from __future__ import print_function
7
8__author__ = 'shenhan@google.com (Han Shen)'
9
10import os
11import random
12import sys
13import unittest
14
15from cros_utils import command_executer
16from binary_search_tool import binary_search_state
17from binary_search_tool import bisect
18
19import common
20import gen_obj
21
22
23def GenObj():
24  obj_num = random.randint(100, 1000)
25  bad_obj_num = random.randint(obj_num / 100, obj_num / 20)
26  if bad_obj_num == 0:
27    bad_obj_num = 1
28  gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)])
29
30
31def CleanObj():
32  os.remove(common.OBJECTS_FILE)
33  os.remove(common.WORKING_SET_FILE)
34  print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE,
35                                         common.WORKING_SET_FILE))
36
37
38class BisectTest(unittest.TestCase):
39  """Tests for bisect.py"""
40
41  def setUp(self):
42    with open('./is_setup', 'w'):
43      pass
44
45    try:
46      os.remove(binary_search_state.STATE_FILE)
47    except OSError:
48      pass
49
50  def tearDown(self):
51    try:
52      os.remove('./is_setup')
53      os.remove(os.readlink(binary_search_state.STATE_FILE))
54      os.remove(binary_search_state.STATE_FILE)
55    except OSError:
56      pass
57
58  class FullBisector(bisect.Bisector):
59    """Test bisector to test bisect.py with"""
60
61    def __init__(self, options, overrides):
62      super(BisectTest.FullBisector, self).__init__(options, overrides)
63
64    def PreRun(self):
65      GenObj()
66      return 0
67
68    def Run(self):
69      return binary_search_state.Run(get_initial_items='./gen_init_list.py',
70                                     switch_to_good='./switch_to_good.py',
71                                     switch_to_bad='./switch_to_bad.py',
72                                     test_script='./is_good.py',
73                                     prune=True,
74                                     file_args=True)
75
76    def PostRun(self):
77      CleanObj()
78      return 0
79
80  def test_full_bisector(self):
81    ret = bisect.Run(self.FullBisector({}, {}))
82    self.assertEquals(ret, 0)
83    self.assertFalse(os.path.exists(common.OBJECTS_FILE))
84    self.assertFalse(os.path.exists(common.WORKING_SET_FILE))
85
86  def check_output(self):
87    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
88        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
89         'tail -n1'))
90    ls = out.splitlines()
91    self.assertEqual(len(ls), 1)
92    line = ls[0]
93
94    _, _, bad_ones = line.partition('Bad items are: ')
95    bad_ones = bad_ones.split()
96    expected_result = common.ReadObjectsFile()
97
98    # Reconstruct objects file from bad_ones and compare
99    actual_result = [0] * len(expected_result)
100    for bad_obj in bad_ones:
101      actual_result[int(bad_obj)] = 1
102
103    self.assertEqual(actual_result, expected_result)
104
105
106class BisectingUtilsTest(unittest.TestCase):
107  """Tests for bisecting tool."""
108
109  def setUp(self):
110    """Generate [100-1000] object files, and 1-5% of which are bad ones."""
111    GenObj()
112
113    with open('./is_setup', 'w'):
114      pass
115
116    try:
117      os.remove(binary_search_state.STATE_FILE)
118    except OSError:
119      pass
120
121  def tearDown(self):
122    """Cleanup temp files."""
123    CleanObj()
124
125    try:
126      os.remove(os.readlink(binary_search_state.STATE_FILE))
127    except OSError:
128      pass
129
130    cleanup_list = ['./is_setup',
131                    binary_search_state.STATE_FILE,
132                    'noinc_prune_bad',
133                    'noinc_prune_good']
134    for f in cleanup_list:
135      if os.path.exists(f):
136        os.remove(f)
137
138  def runTest(self):
139    ret = binary_search_state.Run(get_initial_items='./gen_init_list.py',
140                                  switch_to_good='./switch_to_good.py',
141                                  switch_to_bad='./switch_to_bad.py',
142                                  test_script='./is_good.py',
143                                  prune=True,
144                                  file_args=True)
145    self.assertEquals(ret, 0)
146    self.check_output()
147
148  def test_arg_parse(self):
149    args = ['--get_initial_items', './gen_init_list.py', '--switch_to_good',
150            './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py',
151            '--test_script', './is_good.py', '--prune', '--file_args']
152    ret = binary_search_state.Main(args)
153    self.assertEquals(ret, 0)
154    self.check_output()
155
156  def test_test_setup_script(self):
157    os.remove('./is_setup')
158    with self.assertRaises(AssertionError):
159      ret = binary_search_state.Run(get_initial_items='./gen_init_list.py',
160                                    switch_to_good='./switch_to_good.py',
161                                    switch_to_bad='./switch_to_bad.py',
162                                    test_script='./is_good.py',
163                                    prune=True,
164                                    file_args=True)
165
166    ret = binary_search_state.Run(get_initial_items='./gen_init_list.py',
167                                  switch_to_good='./switch_to_good.py',
168                                  switch_to_bad='./switch_to_bad.py',
169                                  test_script='./is_good.py',
170                                  test_setup_script='./test_setup.py',
171                                  prune=True,
172                                  file_args=True)
173    self.assertEquals(ret, 0)
174    self.check_output()
175
176  def test_bad_test_setup_script(self):
177    with self.assertRaises(AssertionError):
178      binary_search_state.Run(get_initial_items='./gen_init_list.py',
179                              switch_to_good='./switch_to_good.py',
180                              switch_to_bad='./switch_to_bad.py',
181                              test_script='./is_good.py',
182                              test_setup_script='./test_setup_bad.py',
183                              prune=True,
184                              file_args=True)
185
186  def test_bad_save_state(self):
187    state_file = binary_search_state.STATE_FILE
188    hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE)
189
190    with open(state_file, 'w') as f:
191      f.write('test123')
192
193    bss = binary_search_state.MockBinarySearchState()
194    with self.assertRaises(binary_search_state.Error):
195      bss.SaveState()
196
197    with open(state_file, 'r') as f:
198      self.assertEquals(f.read(), 'test123')
199
200    os.remove(state_file)
201
202    # Cleanup generated save state that has no symlink
203    files = os.listdir(os.getcwd())
204    save_states = [x for x in files if x.startswith(hidden_state_file)]
205    _ = [os.remove(x) for x in save_states]
206
207  def test_save_state(self):
208    state_file = binary_search_state.STATE_FILE
209
210    bss = binary_search_state.MockBinarySearchState()
211    bss.SaveState()
212    self.assertTrue(os.path.exists(state_file))
213    first_state = os.readlink(state_file)
214
215    bss.SaveState()
216    second_state = os.readlink(state_file)
217    self.assertTrue(os.path.exists(state_file))
218    self.assertTrue(second_state != first_state)
219    self.assertFalse(os.path.exists(first_state))
220
221    bss.RemoveState()
222    self.assertFalse(os.path.islink(state_file))
223    self.assertFalse(os.path.exists(second_state))
224
225  def test_load_state(self):
226    test_items = [1, 2, 3, 4, 5]
227
228    bss = binary_search_state.MockBinarySearchState()
229    bss.all_items = test_items
230    bss.currently_good_items = set([1, 2, 3])
231    bss.currently_bad_items = set([4, 5])
232    bss.SaveState()
233
234    bss = None
235
236    bss2 = binary_search_state.MockBinarySearchState.LoadState()
237    self.assertEquals(bss2.all_items, test_items)
238    self.assertEquals(bss2.currently_good_items, set([]))
239    self.assertEquals(bss2.currently_bad_items, set([]))
240
241  def test_tmp_cleanup(self):
242    bss = binary_search_state.MockBinarySearchState(
243        get_initial_items='echo "0\n1\n2\n3"',
244        switch_to_good='./switch_tmp.py',
245        file_args=True)
246    bss.SwitchToGood(['0', '1', '2', '3'])
247
248    tmp_file = None
249    with open('tmp_file', 'r') as f:
250      tmp_file = f.read()
251    os.remove('tmp_file')
252
253    self.assertFalse(os.path.exists(tmp_file))
254    ws = common.ReadWorkingSet()
255    for i in range(3):
256      self.assertEquals(ws[i], 42)
257
258  def test_verify_fail(self):
259    bss = binary_search_state.MockBinarySearchState(
260        get_initial_items='./gen_init_list.py',
261        switch_to_good='./switch_to_bad.py',
262        switch_to_bad='./switch_to_good.py',
263        test_script='./is_good.py',
264        prune=True,
265        file_args=True,
266        verify=True)
267    with self.assertRaises(AssertionError):
268      bss.DoVerify()
269
270  def test_early_terminate(self):
271    bss = binary_search_state.MockBinarySearchState(
272        get_initial_items='./gen_init_list.py',
273        switch_to_good='./switch_to_good.py',
274        switch_to_bad='./switch_to_bad.py',
275        test_script='./is_good.py',
276        prune=True,
277        file_args=True,
278        iterations=1)
279    bss.DoSearch()
280    self.assertFalse(bss.found_items)
281
282  def test_no_prune(self):
283    bss = binary_search_state.MockBinarySearchState(
284        get_initial_items='./gen_init_list.py',
285        switch_to_good='./switch_to_good.py',
286        switch_to_bad='./switch_to_bad.py',
287        test_script='./is_good.py',
288        test_setup_script='./test_setup.py',
289        prune=False,
290        file_args=True)
291    bss.DoSearch()
292    self.assertEquals(len(bss.found_items), 1)
293
294    bad_objs = common.ReadObjectsFile()
295    found_obj = int(bss.found_items.pop())
296    self.assertEquals(bad_objs[found_obj], 1)
297
298  def test_set_file(self):
299    binary_search_state.Run(
300        get_initial_items='./gen_init_list.py',
301        switch_to_good='./switch_to_good_set_file.py',
302        switch_to_bad='./switch_to_bad_set_file.py',
303        test_script='./is_good.py',
304        prune=True,
305        file_args=True,
306        verify=True)
307    self.check_output()
308
309  def test_noincremental_prune(self):
310    ret = binary_search_state.Run(
311        get_initial_items='./gen_init_list.py',
312        switch_to_good='./switch_to_good_noinc_prune.py',
313        switch_to_bad='./switch_to_bad_noinc_prune.py',
314        test_script='./is_good_noinc_prune.py',
315        test_setup_script='./test_setup.py',
316        prune=True,
317        noincremental=True,
318        file_args=True,
319        verify=False)
320    self.assertEquals(ret, 0)
321    self.check_output()
322
323  def check_output(self):
324    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
325        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
326         'tail -n1'))
327    ls = out.splitlines()
328    self.assertEqual(len(ls), 1)
329    line = ls[0]
330
331    _, _, bad_ones = line.partition('Bad items are: ')
332    bad_ones = bad_ones.split()
333    expected_result = common.ReadObjectsFile()
334
335    # Reconstruct objects file from bad_ones and compare
336    actual_result = [0] * len(expected_result)
337    for bad_obj in bad_ones:
338      actual_result[int(bad_obj)] = 1
339
340    self.assertEqual(actual_result, expected_result)
341
342
343class BisectStressTest(unittest.TestCase):
344  """Stress tests for bisecting tool."""
345
346  def test_every_obj_bad(self):
347    amt = 25
348    gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)])
349    ret = binary_search_state.Run(get_initial_items='./gen_init_list.py',
350                                  switch_to_good='./switch_to_good.py',
351                                  switch_to_bad='./switch_to_bad.py',
352                                  test_script='./is_good.py',
353                                  prune=True,
354                                  file_args=True,
355                                  verify=False)
356    self.assertEquals(ret, 0)
357    self.check_output()
358
359  def test_every_index_is_bad(self):
360    amt = 25
361    for i in range(amt):
362      obj_list = ['0'] * amt
363      obj_list[i] = '1'
364      obj_list = ','.join(obj_list)
365      gen_obj.Main(['--obj_list', obj_list])
366      ret = binary_search_state.Run(get_initial_items='./gen_init_list.py',
367                                    switch_to_good='./switch_to_good.py',
368                                    switch_to_bad='./switch_to_bad.py',
369                                    test_setup_script='./test_setup.py',
370                                    test_script='./is_good.py',
371                                    prune=True,
372                                    file_args=True)
373      self.assertEquals(ret, 0)
374      self.check_output()
375
376
377  def check_output(self):
378    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
379        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
380         'tail -n1'))
381    ls = out.splitlines()
382    self.assertEqual(len(ls), 1)
383    line = ls[0]
384
385    _, _, bad_ones = line.partition('Bad items are: ')
386    bad_ones = bad_ones.split()
387    expected_result = common.ReadObjectsFile()
388
389    # Reconstruct objects file from bad_ones and compare
390    actual_result = [0] * len(expected_result)
391    for bad_obj in bad_ones:
392      actual_result[int(bad_obj)] = 1
393
394    self.assertEqual(actual_result, expected_result)
395
396
397def Main(argv):
398  num_tests = 2
399  if len(argv) > 1:
400    num_tests = int(argv[1])
401
402  suite = unittest.TestSuite()
403  for _ in range(0, num_tests):
404    suite.addTest(BisectingUtilsTest())
405  suite.addTest(BisectingUtilsTest('test_arg_parse'))
406  suite.addTest(BisectingUtilsTest('test_test_setup_script'))
407  suite.addTest(BisectingUtilsTest('test_bad_test_setup_script'))
408  suite.addTest(BisectingUtilsTest('test_bad_save_state'))
409  suite.addTest(BisectingUtilsTest('test_save_state'))
410  suite.addTest(BisectingUtilsTest('test_load_state'))
411  suite.addTest(BisectingUtilsTest('test_tmp_cleanup'))
412  suite.addTest(BisectingUtilsTest('test_verify_fail'))
413  suite.addTest(BisectingUtilsTest('test_early_terminate'))
414  suite.addTest(BisectingUtilsTest('test_no_prune'))
415  suite.addTest(BisectingUtilsTest('test_set_file'))
416  suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
417  suite.addTest(BisectTest('test_full_bisector'))
418  suite.addTest(BisectStressTest('test_every_obj_bad'))
419  suite.addTest(BisectStressTest('test_every_index_is_bad'))
420  runner = unittest.TextTestRunner()
421  runner.run(suite)
422
423
424if __name__ == '__main__':
425  Main(sys.argv)
426