1"""
2A stress-test of sorts for LLDB's handling of threads in the inferior.
3
4This test sets a breakpoint in the main thread where test parameters (numbers of
5threads) can be adjusted, runs the inferior to that point, and modifies the
6locals that control the event thread counts. This test also sets a breakpoint in
7breakpoint_func (the function executed by each 'breakpoint' thread) and a
8watchpoint on a global modified in watchpoint_func. The inferior is continued
9until exit or a crash takes place, and the number of events seen by LLDB is
10verified to match the expected number of events.
11"""
12
13import os, signal, time
14import unittest2
15import lldb
16from lldbtest import *
17import lldbutil
18
19# ==================================================
20# Dictionary of signal names
21# ==================================================
22signal_names = dict((getattr(signal, n), n) \
23        for n in dir(signal) if n.startswith('SIG') and '_' not in n )
24
25
26class ConcurrentEventsTestCase(TestBase):
27
28    mydir = os.path.join("functionalities", "thread", "concurrent_events")
29
30    #
31    ## Tests for multiple threads that generate a single event.
32    #
33    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
34    @dwarf_test
35    def test_many_breakpoints_dwarf(self):
36        """Test 100 breakpoints from 100 threads."""
37        self.buildDwarf(dictionary=self.getBuildFlags())
38        self.do_thread_actions(num_breakpoint_threads=100)
39
40    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
41    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
42    @dwarf_test
43    def test_many_watchpoints_dwarf(self):
44        """Test 100 watchpoints from 100 threads."""
45        self.buildDwarf(dictionary=self.getBuildFlags())
46        self.do_thread_actions(num_watchpoint_threads=100)
47
48    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
49    @dwarf_test
50    def test_many_signals_dwarf(self):
51        """Test 100 signals from 100 threads."""
52        self.buildDwarf(dictionary=self.getBuildFlags())
53        self.do_thread_actions(num_signal_threads=100)
54
55    @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
56    @dwarf_test
57    def test_many_crash_dwarf(self):
58        """Test 100 threads that cause a segfault."""
59        self.buildDwarf(dictionary=self.getBuildFlags())
60        self.do_thread_actions(num_crash_threads=100)
61
62
63    #
64    ## Tests for concurrent signal and breakpoint
65    #
66    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
67    @dwarf_test
68    def test_signal_break_dwarf(self):
69        """Test signal and a breakpoint in multiple threads."""
70        self.buildDwarf(dictionary=self.getBuildFlags())
71        self.do_thread_actions(num_breakpoint_threads=1, num_signal_threads=1)
72
73    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
74    @dwarf_test
75    def test_delay_signal_break_dwarf(self):
76        """Test (1-second delay) signal and a breakpoint in multiple threads."""
77        self.buildDwarf(dictionary=self.getBuildFlags())
78        self.do_thread_actions(num_breakpoint_threads=1, num_delay_signal_threads=1)
79
80    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
81    @dwarf_test
82    def test_signal_delay_break_dwarf(self):
83        """Test signal and a (1 second delay) breakpoint in multiple threads."""
84        self.buildDwarf(dictionary=self.getBuildFlags())
85        self.do_thread_actions(num_delay_breakpoint_threads=1, num_signal_threads=1)
86
87
88    #
89    ## Tests for concurrent watchpoint and breakpoint
90    #
91    @dwarf_test
92    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
93    def test_watch_break_dwarf(self):
94        """Test watchpoint and a breakpoint in multiple threads."""
95        self.buildDwarf(dictionary=self.getBuildFlags())
96        self.do_thread_actions(num_breakpoint_threads=1, num_watchpoint_threads=1)
97
98    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
99    @dwarf_test
100    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
101    def test_delay_watch_break_dwarf(self):
102        """Test (1-second delay) watchpoint and a breakpoint in multiple threads."""
103        self.buildDwarf(dictionary=self.getBuildFlags())
104        self.do_thread_actions(num_breakpoint_threads=1, num_delay_watchpoint_threads=1)
105
106    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
107    @dwarf_test
108    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
109    def test_watch_break_dwarf(self):
110        """Test watchpoint and a (1 second delay) breakpoint in multiple threads."""
111        self.buildDwarf(dictionary=self.getBuildFlags())
112        self.do_thread_actions(num_delay_breakpoint_threads=1, num_watchpoint_threads=1)
113
114    #
115    ## Tests for concurrent signal and watchpoint
116    #
117    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
118    @dwarf_test
119    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
120    def test_signal_watch_dwarf(self):
121        """Test a watchpoint and a signal in multiple threads."""
122        self.buildDwarf(dictionary=self.getBuildFlags())
123        self.do_thread_actions(num_signal_threads=1, num_watchpoint_threads=1)
124
125    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
126    @dwarf_test
127    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
128    def test_delay_signal_watch_dwarf(self):
129        """Test a watchpoint and a (1 second delay) signal in multiple threads."""
130        self.buildDwarf(dictionary=self.getBuildFlags())
131        self.do_thread_actions(num_delay_signal_threads=1, num_watchpoint_threads=1)
132
133    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
134    @dwarf_test
135    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
136    def test_signal_delay_watch_dwarf(self):
137        """Test a (1 second delay) watchpoint and a signal in multiple threads."""
138        self.buildDwarf(dictionary=self.getBuildFlags())
139        self.do_thread_actions(num_signal_threads=1, num_delay_watchpoint_threads=1)
140
141
142    #
143    ## Tests for multiple breakpoint threads
144    #
145    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
146    @dwarf_test
147    def test_two_breakpoint_threads_dwarf(self):
148        """Test two threads that trigger a breakpoint. """
149        self.buildDwarf(dictionary=self.getBuildFlags())
150        self.do_thread_actions(num_breakpoint_threads=2)
151
152    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
153    @dwarf_test
154    def test_breakpoint_one_delay_breakpoint_threads_dwarf(self):
155        """Test threads that trigger a breakpoint where one thread has a 1 second delay. """
156        self.buildDwarf(dictionary=self.getBuildFlags())
157        self.do_thread_actions(num_breakpoint_threads=1,
158                               num_delay_breakpoint_threads=1)
159
160    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
161    @dwarf_test
162    def test_two_breakpoints_one_signal_dwarf(self):
163        """Test two threads that trigger a breakpoint and one signal thread. """
164        self.buildDwarf(dictionary=self.getBuildFlags())
165        self.do_thread_actions(num_breakpoint_threads=2, num_signal_threads=1)
166
167    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
168    @dwarf_test
169    def test_breakpoint_delay_breakpoint_one_signal_dwarf(self):
170        """Test two threads that trigger a breakpoint (one with a 1 second delay) and one signal thread. """
171        self.buildDwarf(dictionary=self.getBuildFlags())
172        self.do_thread_actions(num_breakpoint_threads=1,
173                               num_delay_breakpoint_threads=1,
174                               num_signal_threads=1)
175
176    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
177    @dwarf_test
178    def test_two_breakpoints_one_delay_signal_dwarf(self):
179        """Test two threads that trigger a breakpoint and one (1 second delay) signal thread. """
180        self.buildDwarf(dictionary=self.getBuildFlags())
181        self.do_thread_actions(num_breakpoint_threads=2, num_delay_signal_threads=1)
182
183    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
184    @dwarf_test
185    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
186    def test_two_breakpoints_one_watchpoint_dwarf(self):
187        """Test two threads that trigger a breakpoint and one watchpoint thread. """
188        self.buildDwarf(dictionary=self.getBuildFlags())
189        self.do_thread_actions(num_breakpoint_threads=2, num_watchpoint_threads=1)
190
191    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
192    @dwarf_test
193    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
194    def test_breakpoints_delayed_breakpoint_one_watchpoint_dwarf(self):
195        """Test a breakpoint, a delayed breakpoint, and one watchpoint thread. """
196        self.buildDwarf(dictionary=self.getBuildFlags())
197        self.do_thread_actions(num_breakpoint_threads=1,
198                               num_delay_breakpoint_threads=1,
199                               num_watchpoint_threads=1)
200
201    #
202    ## Tests for multiple watchpoint threads
203    #
204    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
205    @dwarf_test
206    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
207    def test_two_watchpoint_threads_dwarf(self):
208        """Test two threads that trigger a watchpoint. """
209        self.buildDwarf(dictionary=self.getBuildFlags())
210        self.do_thread_actions(num_watchpoint_threads=2)
211
212    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
213    @dwarf_test
214    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
215    def test_watchpoint_with_delay_waychpoint_threads_dwarf(self):
216        """Test two threads that trigger a watchpoint where one thread has a 1 second delay. """
217        self.buildDwarf(dictionary=self.getBuildFlags())
218        self.do_thread_actions(num_watchpoint_threads=1,
219                               num_delay_watchpoint_threads=1)
220
221    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
222    @dwarf_test
223    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
224    def test_two_watchpoints_one_breakpoint_dwarf(self):
225        """Test two threads that trigger a watchpoint and one breakpoint thread. """
226        self.buildDwarf(dictionary=self.getBuildFlags())
227        self.do_thread_actions(num_watchpoint_threads=2, num_breakpoint_threads=1)
228
229    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
230    @dwarf_test
231    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
232    def test_two_watchpoints_one_delay_breakpoint_dwarf(self):
233        """Test two threads that trigger a watchpoint and one (1 second delay) breakpoint thread. """
234        self.buildDwarf(dictionary=self.getBuildFlags())
235        self.do_thread_actions(num_watchpoint_threads=2, num_delay_breakpoint_threads=1)
236
237    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
238    @dwarf_test
239    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
240    def test_watchpoint_delay_watchpoint_one_breakpoint_dwarf(self):
241        """Test two threads that trigger a watchpoint (one with a 1 second delay) and one breakpoint thread. """
242        self.buildDwarf(dictionary=self.getBuildFlags())
243        self.do_thread_actions(num_watchpoint_threads=1,
244                               num_delay_watchpoint_threads=1,
245                               num_breakpoint_threads=1)
246
247    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
248    @dwarf_test
249    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
250    def test_two_watchpoints_one_signal_dwarf(self):
251        """Test two threads that trigger a watchpoint and one signal thread. """
252        self.buildDwarf(dictionary=self.getBuildFlags())
253        self.do_thread_actions(num_watchpoint_threads=2, num_signal_threads=1)
254
255    #
256    ## Test for watchpoint, signal and breakpoint happening concurrently
257    #
258    @dwarf_test
259    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
260    def test_signal_watch_break_dwarf(self):
261        """Test a signal/watchpoint/breakpoint in multiple threads."""
262        self.buildDwarf(dictionary=self.getBuildFlags())
263        self.do_thread_actions(num_signal_threads=1,
264                               num_watchpoint_threads=1,
265                               num_breakpoint_threads=1)
266
267    @dwarf_test
268    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
269    def test_signal_watch_break_dwarf(self):
270        """Test one signal thread with 5 watchpoint and breakpoint threads."""
271        self.buildDwarf(dictionary=self.getBuildFlags())
272        self.do_thread_actions(num_signal_threads=1,
273                               num_watchpoint_threads=5,
274                               num_breakpoint_threads=5)
275
276    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
277    @dwarf_test
278    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
279    def test_signal_watch_break_dwarf(self):
280        """Test with 5 watchpoint and breakpoint threads."""
281        self.buildDwarf(dictionary=self.getBuildFlags())
282        self.do_thread_actions(num_watchpoint_threads=5,
283                               num_breakpoint_threads=5)
284
285
286    #
287    ## Test for crashing threads happening concurrently with other events
288    #
289    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
290    @dwarf_test
291    def test_crash_with_break_dwarf(self):
292        """ Test a thread that crashes while another thread hits a breakpoint."""
293        self.buildDwarf(dictionary=self.getBuildFlags())
294        self.do_thread_actions(num_crash_threads=1, num_breakpoint_threads=1)
295
296    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
297    @dwarf_test
298    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
299    def test_crash_with_watchpoint_dwarf(self):
300        """ Test a thread that crashes while another thread hits a watchpoint."""
301        self.buildDwarf(dictionary=self.getBuildFlags())
302        self.do_thread_actions(num_crash_threads=1, num_watchpoint_threads=1)
303
304    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
305    @dwarf_test
306    def test_crash_with_signal_dwarf(self):
307        """ Test a thread that crashes while another thread generates a signal."""
308        self.buildDwarf(dictionary=self.getBuildFlags())
309        self.do_thread_actions(num_crash_threads=1, num_signal_threads=1)
310
311    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
312    @dwarf_test
313    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
314    def test_crash_with_watchpoint_breakpoint_signal_dwarf(self):
315        """ Test a thread that crashes while other threads generate a signal and hit a watchpoint and breakpoint. """
316        self.buildDwarf(dictionary=self.getBuildFlags())
317        self.do_thread_actions(num_crash_threads=1,
318                               num_breakpoint_threads=1,
319                               num_signal_threads=1,
320                               num_watchpoint_threads=1)
321
322    @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
323    @dwarf_test
324    @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
325    def test_delayed_crash_with_breakpoint_watchpoint_dwarf(self):
326        """ Test a thread with a delayed crash while other threads hit a watchpoint and a breakpoint. """
327        self.buildDwarf(dictionary=self.getBuildFlags())
328        self.do_thread_actions(num_delay_crash_threads=1,
329                               num_breakpoint_threads=1,
330                               num_watchpoint_threads=1)
331
332    @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
333    @dwarf_test
334    def test_delayed_crash_with_breakpoint_signal_dwarf(self):
335        """ Test a thread with a delayed crash while other threads generate a signal and hit a breakpoint. """
336        self.buildDwarf(dictionary=self.getBuildFlags())
337        self.do_thread_actions(num_delay_crash_threads=1,
338                               num_breakpoint_threads=1,
339                               num_signal_threads=1)
340
341    def setUp(self):
342        # Call super's setUp().
343        TestBase.setUp(self)
344        # Find the line number for our breakpoint.
345        self.filename = 'main.cpp'
346        self.thread_breakpoint_line = line_number(self.filename, '// Set breakpoint here')
347        self.setup_breakpoint_line = line_number(self.filename, '// Break here and adjust num')
348        self.finish_breakpoint_line = line_number(self.filename, '// Break here and verify one thread is active')
349
350    def describe_threads(self):
351        ret = []
352        for x in self.inferior_process:
353            id = x.GetIndexID()
354            reason = x.GetStopReason()
355            status = "stopped" if x.IsStopped() else "running"
356            reason_str = lldbutil.stop_reason_to_str(reason)
357            if reason == lldb.eStopReasonBreakpoint:
358                bpid = x.GetStopReasonDataAtIndex(0)
359                bp = self.inferior_target.FindBreakpointByID(bpid)
360                reason_str = "%s hit %d times" % (lldbutil.get_description(bp), bp.GetHitCount())
361            elif reason == lldb.eStopReasonWatchpoint:
362                watchid = x.GetStopReasonDataAtIndex(0)
363                watch = self.inferior_target.FindWatchpointByID(watchid)
364                reason_str = "%s hit %d times" % (lldbutil.get_description(watch), watch.GetHitCount())
365            elif reason == lldb.eStopReasonSignal:
366                reason_str = "signal %s" % (signal_names[x.GetStopReasonDataAtIndex(0)])
367
368            location = "\t".join([lldbutil.get_description(x.GetFrameAtIndex(i)) for i in range(x.GetNumFrames())])
369            ret.append("thread %d %s due to %s at\n\t%s" % (id, status, reason_str, location))
370        return ret
371
372    def add_breakpoint(self, line, descriptions):
373        """ Adds a breakpoint at self.filename:line and appends its description to descriptions, and
374            returns the LLDB SBBreakpoint object.
375        """
376
377        bpno = lldbutil.run_break_set_by_file_and_line(self, self.filename, line, num_expected_locations=-1)
378        bp = self.inferior_target.FindBreakpointByID(bpno)
379        descriptions.append(": file = 'main.cpp', line = %d" % self.finish_breakpoint_line)
380        return bp
381
382    def inferior_done(self):
383        """ Returns true if the inferior is done executing all the event threads (and is stopped at self.finish_breakpoint,
384            or has terminated execution.
385        """
386        return self.finish_breakpoint.GetHitCount() > 0 or \
387                self.crash_count > 0 or \
388                self.inferior_process.GetState == lldb.eStateExited
389
390    def do_thread_actions(self,
391                          num_breakpoint_threads = 0,
392                          num_signal_threads = 0,
393                          num_watchpoint_threads = 0,
394                          num_crash_threads = 0,
395                          num_delay_breakpoint_threads = 0,
396                          num_delay_signal_threads = 0,
397                          num_delay_watchpoint_threads = 0,
398                          num_delay_crash_threads = 0):
399        """ Sets a breakpoint in the main thread where test parameters (numbers of threads) can be adjusted, runs the inferior
400            to that point, and modifies the locals that control the event thread counts. Also sets a breakpoint in
401            breakpoint_func (the function executed by each 'breakpoint' thread) and a watchpoint on a global modified in
402            watchpoint_func. The inferior is continued until exit or a crash takes place, and the number of events seen by LLDB
403            is verified to match the expected number of events.
404        """
405        exe = os.path.join(os.getcwd(), "a.out")
406        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
407
408        # Get the target
409        self.inferior_target = self.dbg.GetSelectedTarget()
410
411        expected_bps = []
412
413        # Initialize all the breakpoints (main thread/aux thread)
414        self.setup_breakpoint = self.add_breakpoint(self.setup_breakpoint_line, expected_bps)
415        self.finish_breakpoint = self.add_breakpoint(self.finish_breakpoint_line, expected_bps)
416
417        # Set the thread breakpoint
418        if num_breakpoint_threads + num_delay_breakpoint_threads > 0:
419            self.thread_breakpoint = self.add_breakpoint(self.thread_breakpoint_line, expected_bps)
420
421        # Verify breakpoints
422        self.expect("breakpoint list -f", "Breakpoint locations shown correctly", substrs = expected_bps)
423
424        # Run the program.
425        self.runCmd("run", RUN_SUCCEEDED)
426
427        # Check we are at line self.setup_breakpoint
428        self.expect("thread backtrace", STOPPED_DUE_TO_BREAKPOINT,
429            substrs = ["stop reason = breakpoint 1."])
430
431        # Initialize the (single) watchpoint on the global variable (g_watchme)
432        if num_watchpoint_threads + num_delay_watchpoint_threads > 0:
433            self.runCmd("watchpoint set variable g_watchme")
434            for w in self.inferior_target.watchpoint_iter():
435                self.thread_watchpoint = w
436                self.assertTrue("g_watchme" in str(self.thread_watchpoint), "Watchpoint location not shown correctly")
437
438        # Get the process
439        self.inferior_process = self.inferior_target.GetProcess()
440
441        # We should be stopped at the setup site where we can set the number of
442        # threads doing each action (break/crash/signal/watch)
443        self.assertEqual(self.inferior_process.GetNumThreads(), 1, 'Expected to stop before any additional threads are spawned.')
444
445        self.runCmd("expr num_breakpoint_threads=%d" % num_breakpoint_threads)
446        self.runCmd("expr num_crash_threads=%d" % num_crash_threads)
447        self.runCmd("expr num_signal_threads=%d" % num_signal_threads)
448        self.runCmd("expr num_watchpoint_threads=%d" % num_watchpoint_threads)
449
450        self.runCmd("expr num_delay_breakpoint_threads=%d" % num_delay_breakpoint_threads)
451        self.runCmd("expr num_delay_crash_threads=%d" % num_delay_crash_threads)
452        self.runCmd("expr num_delay_signal_threads=%d" % num_delay_signal_threads)
453        self.runCmd("expr num_delay_watchpoint_threads=%d" % num_delay_watchpoint_threads)
454
455        # Continue the inferior so threads are spawned
456        self.runCmd("continue")
457
458        # Make sure we see all the threads. The inferior program's threads all synchronize with a pseudo-barrier; that is,
459        # the inferior program ensures all threads are started and running before any thread triggers its 'event'.
460        num_threads = self.inferior_process.GetNumThreads()
461        expected_num_threads = num_breakpoint_threads + num_delay_breakpoint_threads \
462                             + num_signal_threads + num_delay_signal_threads \
463                             + num_watchpoint_threads + num_delay_watchpoint_threads \
464                             + num_crash_threads + num_delay_crash_threads + 1
465        self.assertEqual(num_threads, expected_num_threads,
466            'Expected to see %d threads, but seeing %d. Details:\n%s' % (expected_num_threads,
467                                                                         num_threads,
468                                                                         "\n\t".join(self.describe_threads())))
469
470        self.signal_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
471        self.crash_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
472
473        # Run to completion (or crash)
474        while not self.inferior_done():
475            if self.TraceOn():
476                self.runCmd("thread backtrace all")
477            self.runCmd("continue")
478            self.signal_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
479            self.crash_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
480
481        if num_crash_threads > 0 or num_delay_crash_threads > 0:
482            # Expecting a crash
483            self.assertTrue(self.crash_count > 0,
484                "Expecting at least one thread to crash. Details: %s" % "\t\n".join(self.describe_threads()))
485
486            # Ensure the zombie process is reaped
487            self.runCmd("process kill")
488
489        elif num_crash_threads == 0 and num_delay_crash_threads == 0:
490            # There should be a single active thread (the main one) which hit the breakpoint after joining
491            self.assertEqual(1, self.finish_breakpoint.GetHitCount(), "Expected main thread (finish) breakpoint to be hit once")
492
493            num_threads = self.inferior_process.GetNumThreads()
494            self.assertEqual(1, num_threads, "Expecting 1 thread but seeing %d. Details:%s" % (num_threads,
495                                                                                             "\n\t".join(self.describe_threads())))
496            self.runCmd("continue")
497
498            # The inferior process should have exited without crashing
499            self.assertEqual(0, self.crash_count, "Unexpected thread(s) in crashed state")
500            self.assertTrue(self.inferior_process.GetState() == lldb.eStateExited, PROCESS_EXITED)
501
502            # Verify the number of actions took place matches expected numbers
503            expected_breakpoint_threads = num_delay_breakpoint_threads + num_breakpoint_threads
504            breakpoint_hit_count = self.thread_breakpoint.GetHitCount() if expected_breakpoint_threads > 0 else 0
505            self.assertEqual(expected_breakpoint_threads, breakpoint_hit_count,
506                "Expected %d breakpoint hits, but got %d" % (expected_breakpoint_threads, breakpoint_hit_count))
507
508            expected_signal_threads = num_delay_signal_threads + num_signal_threads
509            self.assertEqual(expected_signal_threads, self.signal_count,
510                "Expected %d stops due to signal delivery, but got %d" % (expected_signal_threads, self.signal_count))
511
512            expected_watchpoint_threads = num_delay_watchpoint_threads + num_watchpoint_threads
513            watchpoint_hit_count = self.thread_watchpoint.GetHitCount() if expected_watchpoint_threads > 0 else 0
514            self.assertEqual(expected_watchpoint_threads, watchpoint_hit_count,
515                "Expected %d watchpoint hits, got %d" % (expected_watchpoint_threads, watchpoint_hit_count))
516
517
518if __name__ == '__main__':
519    import atexit
520    lldb.SBDebugger.Initialize()
521    atexit.register(lambda: lldb.SBDebugger.Terminate())
522    unittest2.main()
523