1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2019 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Tests for auto bisection of LLVM."""
8
9from __future__ import print_function
10
11import os
12import subprocess
13import time
14import traceback
15import unittest
16import unittest.mock as mock
17
18import auto_llvm_bisection
19import chroot
20import llvm_bisection
21import test_helpers
22
23
24class AutoLLVMBisectionTest(unittest.TestCase):
25  """Unittests for auto bisection of LLVM."""
26
27  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
28  # the script outside of the chroot.
29  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
30  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
31  # by `llvm_bisection.main()` (e.g. network issue, etc.).
32  @mock.patch.object(time, 'sleep')
33  # Simulate behavior of `traceback.print_exc()` when an exception happened in
34  # `llvm_bisection.main()`.
35  @mock.patch.object(traceback, 'print_exc')
36  # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
37  # (exception happened along the way, etc.).
38  @mock.patch.object(llvm_bisection, 'main')
39  # Simulate behavior of `os.path.isfile()` when starting a new bisection.
40  @mock.patch.object(os.path, 'isfile', return_value=False)
41  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
42  # returning the absolute path to that script that updates all 'pending'
43  # tryjobs to the result of `cros buildresult`.
44  @mock.patch.object(
45      auto_llvm_bisection,
46      'GetPathToUpdateAllTryjobsWithAutoScript',
47      return_value='/abs/path/to/update_tryjob.py')
48  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
49  # arguments required by the bisection script.
50  @mock.patch.object(
51      llvm_bisection,
52      'GetCommandLineArgs',
53      return_value=test_helpers.ArgsOutputTest())
54  def testFailedToStartBisection(
55      self, mock_get_args, mock_get_auto_script, mock_is_file,
56      mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot):
57
58    def MockLLVMBisectionRaisesException(_args_output):
59      raise ValueError('Failed to launch more tryjobs.')
60
61    # Use the test function to simulate the behavior of an exception happening
62    # when launching more tryjobs.
63    mock_llvm_bisection.side_effect = MockLLVMBisectionRaisesException
64
65    # Verify the exception is raised when the number of attempts to launched
66    # more tryjobs is exceeded, so unable to continue
67    # bisection.
68    with self.assertRaises(SystemExit) as err:
69      auto_llvm_bisection.main()
70
71    self.assertEqual(err.exception.code, 1)
72
73    mock_outside_chroot.assert_called_once()
74    mock_get_args.assert_called_once()
75    mock_get_auto_script.assert_called_once()
76    self.assertEqual(mock_is_file.call_count, 2)
77    self.assertEqual(mock_llvm_bisection.call_count, 3)
78    self.assertEqual(mock_traceback.call_count, 3)
79    self.assertEqual(mock_sleep.call_count, 2)
80
81  # Simulate the behavior of `subprocess.call()` when successfully updated all
82  # tryjobs whose 'status' value is 'pending'.
83  @mock.patch.object(subprocess, 'call', return_value=0)
84  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
85  # the script outside of the chroot.
86  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
87  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
88  # by `llvm_bisection.main()` (e.g. network issue, etc.).
89  @mock.patch.object(time, 'sleep')
90  # Simulate behavior of `traceback.print_exc()` when an exception happened in
91  # `llvm_bisection.main()`.
92  @mock.patch.object(traceback, 'print_exc')
93  # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
94  # (exception happened along the way, etc.).
95  @mock.patch.object(llvm_bisection, 'main')
96  # Simulate behavior of `os.path.isfile()` when starting a new bisection.
97  @mock.patch.object(os.path, 'isfile')
98  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
99  # returning the absolute path to that script that updates all 'pending'
100  # tryjobs to the result of `cros buildresult`.
101  @mock.patch.object(
102      auto_llvm_bisection,
103      'GetPathToUpdateAllTryjobsWithAutoScript',
104      return_value='/abs/path/to/update_tryjob.py')
105  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
106  # arguments required by the bisection script.
107  @mock.patch.object(
108      llvm_bisection,
109      'GetCommandLineArgs',
110      return_value=test_helpers.ArgsOutputTest())
111  def testSuccessfullyBisectedLLVMRevision(
112      self, mock_get_args, mock_get_auto_script, mock_is_file,
113      mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot,
114      mock_update_tryjobs):
115
116    # Simulate the behavior of `os.path.isfile()` when checking whether the
117    # status file provided exists.
118    @test_helpers.CallCountsToMockFunctions
119    def MockStatusFileCheck(call_count, _last_tested):
120      # Simulate that the status file does not exist, so the LLVM bisection
121      # script would create the status file and launch tryjobs.
122      if call_count < 2:
123        return False
124
125      # Simulate when the status file exists and `subprocess.call()` executes
126      # the script that updates all the 'pending' tryjobs to the result of `cros
127      # buildresult`.
128      if call_count == 2:
129        return True
130
131      assert False, 'os.path.isfile() called more times than expected.'
132
133    # Simulate behavior of `llvm_bisection.main()` when successfully bisected
134    # between the good and bad LLVM revision.
135    @test_helpers.CallCountsToMockFunctions
136    def MockLLVMBisectionReturnValue(call_count, _args_output):
137      # Simulate that successfully launched more tryjobs.
138      if call_count == 0:
139        return 0
140
141      # Simulate that failed to launch more tryjobs.
142      if call_count == 1:
143        raise ValueError('Failed to launch more tryjobs.')
144
145      # Simulate that the bad revision has been found.
146      if call_count == 2:
147        return llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value
148
149      assert False, 'Called `llvm_bisection.main()` more than expected.'
150
151    # Use the test function to simulate the behavior of `llvm_bisection.main()`.
152    mock_llvm_bisection.side_effect = MockLLVMBisectionReturnValue
153
154    # Use the test function to simulate the behavior of `os.path.isfile()`.
155    mock_is_file.side_effect = MockStatusFileCheck
156
157    # Verify the excpetion is raised when successfully found the bad revision.
158    # Uses `sys.exit(0)` to indicate success.
159    with self.assertRaises(SystemExit) as err:
160      auto_llvm_bisection.main()
161
162    self.assertEqual(err.exception.code, 0)
163
164    mock_outside_chroot.assert_called_once()
165    mock_get_args.assert_called_once()
166    mock_get_auto_script.assert_called_once()
167    self.assertEqual(mock_is_file.call_count, 3)
168    self.assertEqual(mock_llvm_bisection.call_count, 3)
169    mock_traceback.assert_called_once()
170    mock_sleep.assert_called_once()
171    mock_update_tryjobs.assert_called_once()
172
173  # Simulate behavior of `subprocess.call()` when failed to update tryjobs to
174  # `cros buildresult` (script failed).
175  @mock.patch.object(subprocess, 'call', return_value=1)
176  # Simulate behavior of `time.time()` when determining the time passed when
177  # updating tryjobs whose 'status' is 'pending'.
178  @mock.patch.object(time, 'time')
179  # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
180  # the script outside of the chroot.
181  @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
182  # Simulate behavior of `time.sleep()` when waiting for errors to settle caused
183  # by `llvm_bisection.main()` (e.g. network issue, etc.).
184  @mock.patch.object(time, 'sleep')
185  # Simulate behavior of `traceback.print_exc()` when resuming bisection.
186  @mock.patch.object(os.path, 'isfile', return_value=True)
187  # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
188  # returning the absolute path to that script that updates all 'pending'
189  # tryjobs to the result of `cros buildresult`.
190  @mock.patch.object(
191      auto_llvm_bisection,
192      'GetPathToUpdateAllTryjobsWithAutoScript',
193      return_value='/abs/path/to/update_tryjob.py')
194  # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
195  # arguments required by the bisection script.
196  @mock.patch.object(
197      llvm_bisection,
198      'GetCommandLineArgs',
199      return_value=test_helpers.ArgsOutputTest())
200  def testFailedToUpdatePendingTryJobs(
201      self, mock_get_args, mock_get_auto_script, mock_is_file, mock_sleep,
202      mock_outside_chroot, mock_time, mock_update_tryjobs):
203
204    # Simulate behavior of `time.time()` for time passed.
205    @test_helpers.CallCountsToMockFunctions
206    def MockTimePassed(call_count):
207      if call_count < 3:
208        return call_count
209
210      assert False, 'Called `time.time()` more than expected.'
211
212    # Use the test function to simulate the behavior of `time.time()`.
213    mock_time.side_effect = MockTimePassed
214
215    # Reduce the polling limit for the test case to terminate faster.
216    auto_llvm_bisection.POLLING_LIMIT_SECS = 1
217
218    # Verify the exception is raised when unable to update tryjobs whose
219    # 'status' value is 'pending'.
220    with self.assertRaises(SystemExit) as err:
221      auto_llvm_bisection.main()
222
223    self.assertEqual(err.exception.code, 1)
224
225    mock_outside_chroot.assert_called_once()
226    mock_get_args.assert_called_once()
227    mock_get_auto_script.assert_called_once()
228    self.assertEqual(mock_is_file.call_count, 2)
229    mock_sleep.assert_called_once()
230    self.assertEqual(mock_time.call_count, 3)
231    self.assertEqual(mock_update_tryjobs.call_count, 2)
232
233
234if __name__ == '__main__':
235  unittest.main()
236