1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2020 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Tests for bisecting tool."""
8
9from __future__ import division
10from __future__ import print_function
11
12__author__ = 'shenhan@google.com (Han Shen)'
13
14import os
15import random
16import sys
17import unittest
18
19from cros_utils import command_executer
20from binary_search_tool import binary_search_state
21from binary_search_tool import run_bisect
22
23from binary_search_tool.test import common
24from binary_search_tool.test import gen_obj
25
26
27def GenObj():
28  obj_num = random.randint(100, 1000)
29  bad_obj_num = random.randint(obj_num // 100, obj_num // 20)
30  if bad_obj_num == 0:
31    bad_obj_num = 1
32  gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)])
33
34
35def CleanObj():
36  os.remove(common.OBJECTS_FILE)
37  os.remove(common.WORKING_SET_FILE)
38  print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE,
39                                         common.WORKING_SET_FILE))
40
41
42class BisectTest(unittest.TestCase):
43  """Tests for run_bisect.py"""
44
45  def setUp(self):
46    with open('./is_setup', 'w', encoding='utf-8'):
47      pass
48
49    try:
50      os.remove(binary_search_state.STATE_FILE)
51    except OSError:
52      pass
53
54  def tearDown(self):
55    try:
56      os.remove('./is_setup')
57      os.remove(os.readlink(binary_search_state.STATE_FILE))
58      os.remove(binary_search_state.STATE_FILE)
59    except OSError:
60      pass
61
62  class FullBisector(run_bisect.Bisector):
63    """Test bisector to test run_bisect.py with"""
64
65    def __init__(self, options, overrides):
66      super(BisectTest.FullBisector, self).__init__(options, overrides)
67
68    def PreRun(self):
69      GenObj()
70      return 0
71
72    def Run(self):
73      return binary_search_state.Run(
74          get_initial_items='./gen_init_list.py',
75          switch_to_good='./switch_to_good.py',
76          switch_to_bad='./switch_to_bad.py',
77          test_script='./is_good.py',
78          prune=True,
79          file_args=True)
80
81    def PostRun(self):
82      CleanObj()
83      return 0
84
85  def test_full_bisector(self):
86    ret = run_bisect.Run(self.FullBisector({}, {}))
87    self.assertEqual(ret, 0)
88    self.assertFalse(os.path.exists(common.OBJECTS_FILE))
89    self.assertFalse(os.path.exists(common.WORKING_SET_FILE))
90
91  def check_output(self):
92    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
93        ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
94         'tail -n1'))
95    ls = out.splitlines()
96    self.assertEqual(len(ls), 1)
97    line = ls[0]
98
99    _, _, bad_ones = line.partition('Bad items are: ')
100    bad_ones = bad_ones.split()
101    expected_result = common.ReadObjectsFile()
102
103    # Reconstruct objects file from bad_ones and compare
104    actual_result = [0] * len(expected_result)
105    for bad_obj in bad_ones:
106      actual_result[int(bad_obj)] = 1
107
108    self.assertEqual(actual_result, expected_result)
109
110
111class BisectingUtilsTest(unittest.TestCase):
112  """Tests for bisecting tool."""
113
114  def setUp(self):
115    """Generate [100-1000] object files, and 1-5% of which are bad ones."""
116    GenObj()
117
118    with open('./is_setup', 'w', encoding='utf-8'):
119      pass
120
121    try:
122      os.remove(binary_search_state.STATE_FILE)
123    except OSError:
124      pass
125
126  def tearDown(self):
127    """Cleanup temp files."""
128    CleanObj()
129
130    try:
131      os.remove(os.readlink(binary_search_state.STATE_FILE))
132    except OSError:
133      pass
134
135    cleanup_list = [
136        './is_setup', binary_search_state.STATE_FILE, 'noinc_prune_bad',
137        'noinc_prune_good', './cmd_script.sh'
138    ]
139    for f in cleanup_list:
140      if os.path.exists(f):
141        os.remove(f)
142
143  def runTest(self):
144    ret = binary_search_state.Run(
145        get_initial_items='./gen_init_list.py',
146        switch_to_good='./switch_to_good.py',
147        switch_to_bad='./switch_to_bad.py',
148        test_script='./is_good.py',
149        prune=True,
150        file_args=True)
151    self.assertEqual(ret, 0)
152    self.check_output()
153
154  def test_arg_parse(self):
155    args = [
156        '--get_initial_items', './gen_init_list.py', '--switch_to_good',
157        './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py',
158        '--test_script', './is_good.py', '--prune', '--file_args'
159    ]
160    ret = binary_search_state.Main(args)
161    self.assertEqual(ret, 0)
162    self.check_output()
163
164  def test_test_setup_script(self):
165    os.remove('./is_setup')
166    with self.assertRaises(AssertionError):
167      ret = binary_search_state.Run(
168          get_initial_items='./gen_init_list.py',
169          switch_to_good='./switch_to_good.py',
170          switch_to_bad='./switch_to_bad.py',
171          test_script='./is_good.py',
172          prune=True,
173          file_args=True)
174
175    ret = binary_search_state.Run(
176        get_initial_items='./gen_init_list.py',
177        switch_to_good='./switch_to_good.py',
178        switch_to_bad='./switch_to_bad.py',
179        test_script='./is_good.py',
180        test_setup_script='./test_setup.py',
181        prune=True,
182        file_args=True)
183    self.assertEqual(ret, 0)
184    self.check_output()
185
186  def test_bad_test_setup_script(self):
187    with self.assertRaises(AssertionError):
188      binary_search_state.Run(
189          get_initial_items='./gen_init_list.py',
190          switch_to_good='./switch_to_good.py',
191          switch_to_bad='./switch_to_bad.py',
192          test_script='./is_good.py',
193          test_setup_script='./test_setup_bad.py',
194          prune=True,
195          file_args=True)
196
197  def test_bad_save_state(self):
198    state_file = binary_search_state.STATE_FILE
199    hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE)
200
201    with open(state_file, 'w', encoding='utf-8') as f:
202      f.write('test123')
203
204    bss = binary_search_state.MockBinarySearchState()
205    with self.assertRaises(OSError):
206      bss.SaveState()
207
208    with open(state_file, 'r', encoding='utf-8') as f:
209      self.assertEqual(f.read(), 'test123')
210
211    os.remove(state_file)
212
213    # Cleanup generated save state that has no symlink
214    files = os.listdir(os.getcwd())
215    save_states = [x for x in files if x.startswith(hidden_state_file)]
216    _ = [os.remove(x) for x in save_states]
217
218  def test_save_state(self):
219    state_file = binary_search_state.STATE_FILE
220
221    bss = binary_search_state.MockBinarySearchState()
222    bss.SaveState()
223    self.assertTrue(os.path.exists(state_file))
224    first_state = os.readlink(state_file)
225
226    bss.SaveState()
227    second_state = os.readlink(state_file)
228    self.assertTrue(os.path.exists(state_file))
229    self.assertTrue(second_state != first_state)
230    self.assertFalse(os.path.exists(first_state))
231
232    bss.RemoveState()
233    self.assertFalse(os.path.islink(state_file))
234    self.assertFalse(os.path.exists(second_state))
235
236  def test_load_state(self):
237    test_items = [1, 2, 3, 4, 5]
238
239    bss = binary_search_state.MockBinarySearchState()
240    bss.all_items = test_items
241    bss.currently_good_items = set([1, 2, 3])
242    bss.currently_bad_items = set([4, 5])
243    bss.SaveState()
244
245    bss = None
246
247    bss2 = binary_search_state.MockBinarySearchState.LoadState()
248    self.assertEqual(bss2.all_items, test_items)
249    self.assertEqual(bss2.currently_good_items, set([]))
250    self.assertEqual(bss2.currently_bad_items, set([]))
251
252  def test_tmp_cleanup(self):
253    bss = binary_search_state.MockBinarySearchState(
254        get_initial_items='echo "0\n1\n2\n3"',
255        switch_to_good='./switch_tmp.py',
256        file_args=True)
257    bss.SwitchToGood(['0', '1', '2', '3'])
258
259    tmp_file = None
260    with open('tmp_file', 'r', encoding='utf-8') as f:
261      tmp_file = f.read()
262    os.remove('tmp_file')
263
264    self.assertFalse(os.path.exists(tmp_file))
265    ws = common.ReadWorkingSet()
266    for i in range(3):
267      self.assertEqual(ws[i], 42)
268
269  def test_verify_fail(self):
270    bss = binary_search_state.MockBinarySearchState(
271        get_initial_items='./gen_init_list.py',
272        switch_to_good='./switch_to_bad.py',
273        switch_to_bad='./switch_to_good.py',
274        test_script='./is_good.py',
275        prune=True,
276        file_args=True,
277        verify=True)
278    with self.assertRaises(AssertionError):
279      bss.DoVerify()
280
281  def test_early_terminate(self):
282    bss = binary_search_state.MockBinarySearchState(
283        get_initial_items='./gen_init_list.py',
284        switch_to_good='./switch_to_good.py',
285        switch_to_bad='./switch_to_bad.py',
286        test_script='./is_good.py',
287        prune=True,
288        file_args=True,
289        iterations=1)
290    bss.DoSearchBadItems()
291    self.assertFalse(bss.found_items)
292
293  def test_no_prune(self):
294    bss = binary_search_state.MockBinarySearchState(
295        get_initial_items='./gen_init_list.py',
296        switch_to_good='./switch_to_good.py',
297        switch_to_bad='./switch_to_bad.py',
298        test_script='./is_good.py',
299        test_setup_script='./test_setup.py',
300        prune=False,
301        file_args=True)
302    bss.DoSearchBadItems()
303    self.assertEqual(len(bss.found_items), 1)
304
305    bad_objs = common.ReadObjectsFile()
306    found_obj = int(bss.found_items.pop())
307    self.assertEqual(bad_objs[found_obj], 1)
308
309  def test_set_file(self):
310    binary_search_state.Run(
311        get_initial_items='./gen_init_list.py',
312        switch_to_good='./switch_to_good_set_file.py',
313        switch_to_bad='./switch_to_bad_set_file.py',
314        test_script='./is_good.py',
315        prune=True,
316        file_args=True,
317        verify=True)
318    self.check_output()
319
320  def test_noincremental_prune(self):
321    ret = binary_search_state.Run(
322        get_initial_items='./gen_init_list.py',
323        switch_to_good='./switch_to_good_noinc_prune.py',
324        switch_to_bad='./switch_to_bad_noinc_prune.py',
325        test_script='./is_good_noinc_prune.py',
326        test_setup_script='./test_setup.py',
327        prune=True,
328        noincremental=True,
329        file_args=True,
330        verify=False)
331    self.assertEqual(ret, 0)
332    self.check_output()
333
334  def check_output(self):
335    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
336        ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
337         'tail -n1'))
338    ls = out.splitlines()
339    self.assertEqual(len(ls), 1)
340    line = ls[0]
341
342    _, _, bad_ones = line.partition('Bad items are: ')
343    bad_ones = bad_ones.split()
344    expected_result = common.ReadObjectsFile()
345
346    # Reconstruct objects file from bad_ones and compare
347    actual_result = [0] * len(expected_result)
348    for bad_obj in bad_ones:
349      actual_result[int(bad_obj)] = 1
350
351    self.assertEqual(actual_result, expected_result)
352
353
354class BisectingUtilsPassTest(BisectingUtilsTest):
355  """Tests for bisecting tool at pass/transformation level."""
356
357  def check_pass_output(self, pass_name, pass_num, trans_num):
358    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
359        ('grep "Bad pass: " logs/binary_search_tool_test.py.out | '
360         'tail -n1'))
361    ls = out.splitlines()
362    self.assertEqual(len(ls), 1)
363    line = ls[0]
364    _, _, bad_info = line.partition('Bad pass: ')
365    actual_info = pass_name + ' at number ' + str(pass_num)
366    self.assertEqual(actual_info, bad_info)
367
368    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
369        ('grep "Bad transformation number: '
370         '" logs/binary_search_tool_test.py.out | '
371         'tail -n1'))
372    ls = out.splitlines()
373    self.assertEqual(len(ls), 1)
374    line = ls[0]
375    _, _, bad_info = line.partition('Bad transformation number: ')
376    actual_info = str(trans_num)
377    self.assertEqual(actual_info, bad_info)
378
379  def test_with_prune(self):
380    ret = binary_search_state.Run(
381        get_initial_items='./gen_init_list.py',
382        switch_to_good='./switch_to_good.py',
383        switch_to_bad='./switch_to_bad.py',
384        test_script='./is_good.py',
385        pass_bisect='./generate_cmd.py',
386        prune=True,
387        file_args=True)
388    self.assertEqual(ret, 1)
389
390  def test_gen_cmd_script(self):
391    bss = binary_search_state.MockBinarySearchState(
392        get_initial_items='./gen_init_list.py',
393        switch_to_good='./switch_to_good.py',
394        switch_to_bad='./switch_to_bad.py',
395        test_script='./is_good.py',
396        pass_bisect='./generate_cmd.py',
397        prune=False,
398        file_args=True)
399    bss.DoSearchBadItems()
400    cmd_script_path = bss.cmd_script
401    self.assertTrue(os.path.exists(cmd_script_path))
402
403  def test_no_pass_support(self):
404    bss = binary_search_state.MockBinarySearchState(
405        get_initial_items='./gen_init_list.py',
406        switch_to_good='./switch_to_good.py',
407        switch_to_bad='./switch_to_bad.py',
408        test_script='./is_good.py',
409        pass_bisect='./generate_cmd.py',
410        prune=False,
411        file_args=True)
412    bss.cmd_script = './cmd_script_no_support.py'
413    # No support for -opt-bisect-limit
414    with self.assertRaises(RuntimeError):
415      bss.BuildWithPassLimit(-1)
416
417  def test_no_transform_support(self):
418    bss = binary_search_state.MockBinarySearchState(
419        get_initial_items='./gen_init_list.py',
420        switch_to_good='./switch_to_good.py',
421        switch_to_bad='./switch_to_bad.py',
422        test_script='./is_good.py',
423        pass_bisect='./generate_cmd.py',
424        prune=False,
425        file_args=True)
426    bss.cmd_script = './cmd_script_no_support.py'
427    # No support for -print-debug-counter
428    with self.assertRaises(RuntimeError):
429      bss.BuildWithTransformLimit(-1, 'counter_name')
430
431  def test_pass_transform_bisect(self):
432    bss = binary_search_state.MockBinarySearchState(
433        get_initial_items='./gen_init_list.py',
434        switch_to_good='./switch_to_good.py',
435        switch_to_bad='./switch_to_bad.py',
436        test_script='./is_good.py',
437        pass_bisect='./generate_cmd.py',
438        prune=False,
439        file_args=True)
440    pass_num = 4
441    trans_num = 19
442    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
443    bss.DoSearchBadPass()
444    self.check_pass_output('instcombine-visit', pass_num, trans_num)
445
446  def test_result_not_reproduced_pass(self):
447    bss = binary_search_state.MockBinarySearchState(
448        get_initial_items='./gen_init_list.py',
449        switch_to_good='./switch_to_good.py',
450        switch_to_bad='./switch_to_bad.py',
451        test_script='./is_good.py',
452        pass_bisect='./generate_cmd.py',
453        prune=False,
454        file_args=True)
455    # Fails reproducing at pass level.
456    pass_num = 0
457    trans_num = 19
458    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
459    with self.assertRaises(ValueError):
460      bss.DoSearchBadPass()
461
462  def test_result_not_reproduced_transform(self):
463    bss = binary_search_state.MockBinarySearchState(
464        get_initial_items='./gen_init_list.py',
465        switch_to_good='./switch_to_good.py',
466        switch_to_bad='./switch_to_bad.py',
467        test_script='./is_good.py',
468        pass_bisect='./generate_cmd.py',
469        prune=False,
470        file_args=True)
471    # Fails reproducing at transformation level.
472    pass_num = 4
473    trans_num = 0
474    bss.cmd_script = './cmd_script.py %d %d' % (pass_num, trans_num)
475    with self.assertRaises(ValueError):
476      bss.DoSearchBadPass()
477
478
479class BisectStressTest(unittest.TestCase):
480  """Stress tests for bisecting tool."""
481
482  def test_every_obj_bad(self):
483    amt = 25
484    gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)])
485    ret = binary_search_state.Run(
486        get_initial_items='./gen_init_list.py',
487        switch_to_good='./switch_to_good.py',
488        switch_to_bad='./switch_to_bad.py',
489        test_script='./is_good.py',
490        prune=True,
491        file_args=True,
492        verify=False)
493    self.assertEqual(ret, 0)
494    self.check_output()
495
496  def test_every_index_is_bad(self):
497    amt = 25
498    for i in range(amt):
499      obj_list = ['0'] * amt
500      obj_list[i] = '1'
501      obj_list = ','.join(obj_list)
502      gen_obj.Main(['--obj_list', obj_list])
503      ret = binary_search_state.Run(
504          get_initial_items='./gen_init_list.py',
505          switch_to_good='./switch_to_good.py',
506          switch_to_bad='./switch_to_bad.py',
507          test_setup_script='./test_setup.py',
508          test_script='./is_good.py',
509          prune=True,
510          file_args=True)
511      self.assertEqual(ret, 0)
512      self.check_output()
513
514  def check_output(self):
515    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
516        ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
517         'tail -n1'))
518    ls = out.splitlines()
519    self.assertEqual(len(ls), 1)
520    line = ls[0]
521
522    _, _, bad_ones = line.partition('Bad items are: ')
523    bad_ones = bad_ones.split()
524    expected_result = common.ReadObjectsFile()
525
526    # Reconstruct objects file from bad_ones and compare
527    actual_result = [0] * len(expected_result)
528    for bad_obj in bad_ones:
529      actual_result[int(bad_obj)] = 1
530
531    self.assertEqual(actual_result, expected_result)
532
533
534def Main(argv):
535  num_tests = 2
536  if len(argv) > 1:
537    num_tests = int(argv[1])
538
539  suite = unittest.TestSuite()
540  for _ in range(0, num_tests):
541    suite.addTest(BisectingUtilsTest())
542  suite.addTest(BisectingUtilsTest('test_arg_parse'))
543  suite.addTest(BisectingUtilsTest('test_test_setup_script'))
544  suite.addTest(BisectingUtilsTest('test_bad_test_setup_script'))
545  suite.addTest(BisectingUtilsTest('test_bad_save_state'))
546  suite.addTest(BisectingUtilsTest('test_save_state'))
547  suite.addTest(BisectingUtilsTest('test_load_state'))
548  suite.addTest(BisectingUtilsTest('test_tmp_cleanup'))
549  suite.addTest(BisectingUtilsTest('test_verify_fail'))
550  suite.addTest(BisectingUtilsTest('test_early_terminate'))
551  suite.addTest(BisectingUtilsTest('test_no_prune'))
552  suite.addTest(BisectingUtilsTest('test_set_file'))
553  suite.addTest(BisectingUtilsTest('test_noincremental_prune'))
554  suite.addTest(BisectingUtilsPassTest('test_with_prune'))
555  suite.addTest(BisectingUtilsPassTest('test_gen_cmd_script'))
556  suite.addTest(BisectingUtilsPassTest('test_no_pass_support'))
557  suite.addTest(BisectingUtilsPassTest('test_no_transform_support'))
558  suite.addTest(BisectingUtilsPassTest('test_pass_transform_bisect'))
559  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_pass'))
560  suite.addTest(BisectingUtilsPassTest('test_result_not_reproduced_transform'))
561  suite.addTest(BisectTest('test_full_bisector'))
562  suite.addTest(BisectStressTest('test_every_obj_bad'))
563  suite.addTest(BisectStressTest('test_every_index_is_bad'))
564  runner = unittest.TextTestRunner()
565  runner.run(suite)
566
567
568if __name__ == '__main__':
569  Main(sys.argv)
570