1#!/usr/bin/env python2
2
3# Copyright 2018 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Tests for bisecting tool."""
7
8from __future__ import print_function
9
10__author__ = 'shenhan@google.com (Han Shen)'
11
12import os
13import random
14import sys
15import unittest
16
17from cros_utils import command_executer
18from binary_search_tool import binary_search_state
19from binary_search_tool import bisect
20
21import common
22import gen_obj
23
24
25def GenObj():
26  obj_num = random.randint(100, 1000)
27  bad_obj_num = random.randint(obj_num / 100, obj_num / 20)
28  if bad_obj_num == 0:
29    bad_obj_num = 1
30  gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)])
31
32
33def CleanObj():
34  os.remove(common.OBJECTS_FILE)
35  os.remove(common.WORKING_SET_FILE)
36  print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE,
37                                         common.WORKING_SET_FILE))
38
39
40class BisectTest(unittest.TestCase):
41  """Tests for bisect.py"""
42
43  def setUp(self):
44    with open('./is_setup', 'w'):
45      pass
46
47    try:
48      os.remove(binary_search_state.STATE_FILE)
49    except OSError:
50      pass
51
52  def tearDown(self):
53    try:
54      os.remove('./is_setup')
55      os.remove(os.readlink(binary_search_state.STATE_FILE))
56      os.remove(binary_search_state.STATE_FILE)
57    except OSError:
58      pass
59
60  class FullBisector(bisect.Bisector):
61    """Test bisector to test bisect.py with"""
62
63    def __init__(self, options, overrides):
64      super(BisectTest.FullBisector, self).__init__(options, overrides)
65
66    def PreRun(self):
67      GenObj()
68      return 0
69
70    def Run(self):
71      return binary_search_state.Run(
72          get_initial_items='./gen_init_list.py',
73          switch_to_good='./switch_to_good.py',
74          switch_to_bad='./switch_to_bad.py',
75          test_script='./is_good.py',
76          prune=True,
77          file_args=True)
78
79    def PostRun(self):
80      CleanObj()
81      return 0
82
83  def test_full_bisector(self):
84    ret = bisect.Run(self.FullBisector({}, {}))
85    self.assertEquals(ret, 0)
86    self.assertFalse(os.path.exists(common.OBJECTS_FILE))
87    self.assertFalse(os.path.exists(common.WORKING_SET_FILE))
88
89  def check_output(self):
90    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
91        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
92         'tail -n1'))
93    ls = out.splitlines()
94    self.assertEqual(len(ls), 1)
95    line = ls[0]
96
97    _, _, bad_ones = line.partition('Bad items are: ')
98    bad_ones = bad_ones.split()
99    expected_result = common.ReadObjectsFile()
100
101    # Reconstruct objects file from bad_ones and compare
102    actual_result = [0] * len(expected_result)
103    for bad_obj in bad_ones:
104      actual_result[int(bad_obj)] = 1
105
106    self.assertEqual(actual_result, expected_result)
107
108
109class BisectingUtilsTest(unittest.TestCase):
110  """Tests for bisecting tool."""
111
112  def setUp(self):
113    """Generate [100-1000] object files, and 1-5% of which are bad ones."""
114    GenObj()
115
116    with open('./is_setup', 'w'):
117      pass
118
119    try:
120      os.remove(binary_search_state.STATE_FILE)
121    except OSError:
122      pass
123
124  def tearDown(self):
125    """Cleanup temp files."""
126    CleanObj()
127
128    try:
129      os.remove(os.readlink(binary_search_state.STATE_FILE))
130    except OSError:
131      pass
132
133    cleanup_list = [
134        './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad',
135        'noinc_prune_good'
136    ]
137    for f in cleanup_list:
138      if os.path.exists(f):
139        os.remove(f)
140
141  def runTest(self):
142    ret = binary_search_state.Run(
143        get_initial_items='./gen_init_list.py',
144        switch_to_good='./switch_to_good.py',
145        switch_to_bad='./switch_to_bad.py',
146        test_script='./is_good.py',
147        prune=True,
148        file_args=True)
149    self.assertEquals(ret, 0)
150    self.check_output()
151
152  def test_arg_parse(self):
153    args = [
154        '--get_initial_items', './gen_init_list.py', '--switch_to_good',
155        './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py',
156        '--test_script', './is_good.py', '--prune', '--file_args'
157    ]
158    ret = binary_search_state.Main(args)
159    self.assertEquals(ret, 0)
160    self.check_output()
161
162  def test_test_setup_script(self):
163    os.remove('./is_setup')
164    with self.assertRaises(AssertionError):
165      ret = binary_search_state.Run(
166          get_initial_items='./gen_init_list.py',
167          switch_to_good='./switch_to_good.py',
168          switch_to_bad='./switch_to_bad.py',
169          test_script='./is_good.py',
170          prune=True,
171          file_args=True)
172
173    ret = binary_search_state.Run(
174        get_initial_items='./gen_init_list.py',
175        switch_to_good='./switch_to_good.py',
176        switch_to_bad='./switch_to_bad.py',
177        test_script='./is_good.py',
178        test_setup_script='./test_setup.py',
179        prune=True,
180        file_args=True)
181    self.assertEquals(ret, 0)
182    self.check_output()
183
184  def test_bad_test_setup_script(self):
185    with self.assertRaises(AssertionError):
186      binary_search_state.Run(
187          get_initial_items='./gen_init_list.py',
188          switch_to_good='./switch_to_good.py',
189          switch_to_bad='./switch_to_bad.py',
190          test_script='./is_good.py',
191          test_setup_script='./test_setup_bad.py',
192          prune=True,
193          file_args=True)
194
195  def test_bad_save_state(self):
196    state_file = binary_search_state.STATE_FILE
197    hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE)
198
199    with open(state_file, 'w') as f:
200      f.write('test123')
201
202    bss = binary_search_state.MockBinarySearchState()
203    with self.assertRaises(binary_search_state.Error):
204      bss.SaveState()
205
206    with open(state_file, 'r') as f:
207      self.assertEquals(f.read(), 'test123')
208
209    os.remove(state_file)
210
211    # Cleanup generated save state that has no symlink
212    files = os.listdir(os.getcwd())
213    save_states = [x for x in files if x.startswith(hidden_state_file)]
214    _ = [os.remove(x) for x in save_states]
215
216  def test_save_state(self):
217    state_file = binary_search_state.STATE_FILE
218
219    bss = binary_search_state.MockBinarySearchState()
220    bss.SaveState()
221    self.assertTrue(os.path.exists(state_file))
222    first_state = os.readlink(state_file)
223
224    bss.SaveState()
225    second_state = os.readlink(state_file)
226    self.assertTrue(os.path.exists(state_file))
227    self.assertTrue(second_state != first_state)
228    self.assertFalse(os.path.exists(first_state))
229
230    bss.RemoveState()
231    self.assertFalse(os.path.islink(state_file))
232    self.assertFalse(os.path.exists(second_state))
233
234  def test_load_state(self):
235    test_items = [1, 2, 3, 4, 5]
236
237    bss = binary_search_state.MockBinarySearchState()
238    bss.all_items = test_items
239    bss.currently_good_items = set([1, 2, 3])
240    bss.currently_bad_items = set([4, 5])
241    bss.SaveState()
242
243    bss = None
244
245    bss2 = binary_search_state.MockBinarySearchState.LoadState()
246    self.assertEquals(bss2.all_items, test_items)
247    self.assertEquals(bss2.currently_good_items, set([]))
248    self.assertEquals(bss2.currently_bad_items, set([]))
249
250  def test_tmp_cleanup(self):
251    bss = binary_search_state.MockBinarySearchState(
252        get_initial_items='echo "0\n1\n2\n3"',
253        switch_to_good='./switch_tmp.py',
254        file_args=True)
255    bss.SwitchToGood(['0', '1', '2', '3'])
256
257    tmp_file = None
258    with open('tmp_file', 'r') as f:
259      tmp_file = f.read()
260    os.remove('tmp_file')
261
262    self.assertFalse(os.path.exists(tmp_file))
263    ws = common.ReadWorkingSet()
264    for i in range(3):
265      self.assertEquals(ws[i], 42)
266
267  def test_verify_fail(self):
268    bss = binary_search_state.MockBinarySearchState(
269        get_initial_items='./gen_init_list.py',
270        switch_to_good='./switch_to_bad.py',
271        switch_to_bad='./switch_to_good.py',
272        test_script='./is_good.py',
273        prune=True,
274        file_args=True,
275        verify=True)
276    with self.assertRaises(AssertionError):
277      bss.DoVerify()
278
279  def test_early_terminate(self):
280    bss = binary_search_state.MockBinarySearchState(
281        get_initial_items='./gen_init_list.py',
282        switch_to_good='./switch_to_good.py',
283        switch_to_bad='./switch_to_bad.py',
284        test_script='./is_good.py',
285        prune=True,
286        file_args=True,
287        iterations=1)
288    bss.DoSearchBadItems()
289    self.assertFalse(bss.found_items)
290
291  def test_no_prune(self):
292    bss = binary_search_state.MockBinarySearchState(
293        get_initial_items='./gen_init_list.py',
294        switch_to_good='./switch_to_good.py',
295        switch_to_bad='./switch_to_bad.py',
296        test_script='./is_good.py',
297        test_setup_script='./test_setup.py',
298        prune=False,
299        file_args=True)
300    bss.DoSearchBadItems()
301    self.assertEquals(len(bss.found_items), 1)
302
303    bad_objs = common.ReadObjectsFile()
304    found_obj = int(bss.found_items.pop())
305    self.assertEquals(bad_objs[found_obj], 1)
306
307  def test_pass_bisect(self):
308    bss = binary_search_state.MockBinarySearchState(
309        get_initial_items='./gen_init_list.py',
310        switch_to_good='./switch_to_good.py',
311        switch_to_bad='./switch_to_bad.py',
312        pass_bisect='./generate_cmd.py',
313        test_script='./is_good.py',
314        test_setup_script='./test_setup.py',
315        prune=False,
316        file_args=True)
317    # TODO: Need to design unit tests for pass level bisection
318    bss.DoSearchBadItems()
319    self.assertEquals(len(bss.found_items), 1)
320
321    bad_objs = common.ReadObjectsFile()
322    found_obj = int(bss.found_items.pop())
323    self.assertEquals(bad_objs[found_obj], 1)
324
325  def test_set_file(self):
326    binary_search_state.Run(
327        get_initial_items='./gen_init_list.py',
328        switch_to_good='./switch_to_good_set_file.py',
329        switch_to_bad='./switch_to_bad_set_file.py',
330        test_script='./is_good.py',
331        prune=True,
332        file_args=True,
333        verify=True)
334    self.check_output()
335
336  def test_noincremental_prune(self):
337    ret = binary_search_state.Run(
338        get_initial_items='./gen_init_list.py',
339        switch_to_good='./switch_to_good_noinc_prune.py',
340        switch_to_bad='./switch_to_bad_noinc_prune.py',
341        test_script='./is_good_noinc_prune.py',
342        test_setup_script='./test_setup.py',
343        prune=True,
344        noincremental=True,
345        file_args=True,
346        verify=False)
347    self.assertEquals(ret, 0)
348    self.check_output()
349
350  def check_output(self):
351    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
352        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
353         'tail -n1'))
354    ls = out.splitlines()
355    self.assertEqual(len(ls), 1)
356    line = ls[0]
357
358    _, _, bad_ones = line.partition('Bad items are: ')
359    bad_ones = bad_ones.split()
360    expected_result = common.ReadObjectsFile()
361
362    # Reconstruct objects file from bad_ones and compare
363    actual_result = [0] * len(expected_result)
364    for bad_obj in bad_ones:
365      actual_result[int(bad_obj)] = 1
366
367    self.assertEqual(actual_result, expected_result)
368
369
370class BisectStressTest(unittest.TestCase):
371  """Stress tests for bisecting tool."""
372
373  def test_every_obj_bad(self):
374    amt = 25
375    gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)])
376    ret = binary_search_state.Run(
377        get_initial_items='./gen_init_list.py',
378        switch_to_good='./switch_to_good.py',
379        switch_to_bad='./switch_to_bad.py',
380        test_script='./is_good.py',
381        prune=True,
382        file_args=True,
383        verify=False)
384    self.assertEquals(ret, 0)
385    self.check_output()
386
387  def test_every_index_is_bad(self):
388    amt = 25
389    for i in range(amt):
390      obj_list = ['0'] * amt
391      obj_list[i] = '1'
392      obj_list = ','.join(obj_list)
393      gen_obj.Main(['--obj_list', obj_list])
394      ret = binary_search_state.Run(
395          get_initial_items='./gen_init_list.py',
396          switch_to_good='./switch_to_good.py',
397          switch_to_bad='./switch_to_bad.py',
398          test_setup_script='./test_setup.py',
399          test_script='./is_good.py',
400          prune=True,
401          file_args=True)
402      self.assertEquals(ret, 0)
403      self.check_output()
404
405  def check_output(self):
406    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
407        ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
408         'tail -n1'))
409    ls = out.splitlines()
410    self.assertEqual(len(ls), 1)
411    line = ls[0]
412
413    _, _, bad_ones = line.partition('Bad items are: ')
414    bad_ones = bad_ones.split()
415    expected_result = common.ReadObjectsFile()
416
417    # Reconstruct objects file from bad_ones and compare
418    actual_result = [0] * len(expected_result)
419    for bad_obj in bad_ones:
420      actual_result[int(bad_obj)] = 1
421
422    self.assertEqual(actual_result, expected_result)
423
424
425def Main(argv):
426  num_tests = 2
427  if len(argv) > 1:
428    num_tests = int(argv[1])
429
430  suite = unittest.TestSuite()
431  for _ in range(0, num_tests):
432    suite.addTest(BisectingUtilsTest())
433  suite.addTest(BisectingUtilsTest('test_arg_parse'))
434  suite.addTest(BisectingUtilsTest('test_test_setup_script'))
435  suite.addTest(BisectingUtilsTest('test_bad_test_setup_script'))
436  suite.addTest(BisectingUtilsTest('test_bad_save_state'))
437  suite.addTest(BisectingUtilsTest('test_save_state'))
438  suite.addTest(BisectingUtilsTest('test_load_state'))
439  suite.addTest(BisectingUtilsTest('test_tmp_cleanup'))
440  suite.addTest(BisectingUtilsTest('test_verify_fail'))
441  suite.addTest(BisectingUtilsTest('test_early_terminate'))
442  suite.addTest(BisectingUtilsTest('test_no_prune'))
443  suite.addTest(BisectingUtilsTest('test_set_file'))
444  suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
445  suite.addTest(BisectTest('test_full_bisector'))
446  suite.addTest(BisectStressTest('test_every_obj_bad'))
447  suite.addTest(BisectStressTest('test_every_index_is_bad'))
448  runner = unittest.TextTestRunner()
449  runner.run(suite)
450
451
452if __name__ == '__main__':
453  Main(sys.argv)
454