1import io
2import sys
3import textwrap
4
5from test import support
6
7import traceback
8import unittest
9
10
11class MockTraceback(object):
12    class TracebackException:
13        def __init__(self, *args, **kwargs):
14            self.capture_locals = kwargs.get('capture_locals', False)
15        def format(self):
16            result = ['A traceback']
17            if self.capture_locals:
18                result.append('locals')
19            return result
20
21def restore_traceback():
22    unittest.result.traceback = traceback
23
24
25class Test_TestResult(unittest.TestCase):
26    # Note: there are not separate tests for TestResult.wasSuccessful(),
27    # TestResult.errors, TestResult.failures, TestResult.testsRun or
28    # TestResult.shouldStop because these only have meaning in terms of
29    # other TestResult methods.
30    #
31    # Accordingly, tests for the aforenamed attributes are incorporated
32    # in with the tests for the defining methods.
33    ################################################################
34
35    def test_init(self):
36        result = unittest.TestResult()
37
38        self.assertTrue(result.wasSuccessful())
39        self.assertEqual(len(result.errors), 0)
40        self.assertEqual(len(result.failures), 0)
41        self.assertEqual(result.testsRun, 0)
42        self.assertEqual(result.shouldStop, False)
43        self.assertIsNone(result._stdout_buffer)
44        self.assertIsNone(result._stderr_buffer)
45
46    # "This method can be called to signal that the set of tests being
47    # run should be aborted by setting the TestResult's shouldStop
48    # attribute to True."
49    def test_stop(self):
50        result = unittest.TestResult()
51
52        result.stop()
53
54        self.assertEqual(result.shouldStop, True)
55
56    # "Called when the test case test is about to be run. The default
57    # implementation simply increments the instance's testsRun counter."
58    def test_startTest(self):
59        class Foo(unittest.TestCase):
60            def test_1(self):
61                pass
62
63        test = Foo('test_1')
64
65        result = unittest.TestResult()
66
67        result.startTest(test)
68
69        self.assertTrue(result.wasSuccessful())
70        self.assertEqual(len(result.errors), 0)
71        self.assertEqual(len(result.failures), 0)
72        self.assertEqual(result.testsRun, 1)
73        self.assertEqual(result.shouldStop, False)
74
75        result.stopTest(test)
76
77    # "Called after the test case test has been executed, regardless of
78    # the outcome. The default implementation does nothing."
79    def test_stopTest(self):
80        class Foo(unittest.TestCase):
81            def test_1(self):
82                pass
83
84        test = Foo('test_1')
85
86        result = unittest.TestResult()
87
88        result.startTest(test)
89
90        self.assertTrue(result.wasSuccessful())
91        self.assertEqual(len(result.errors), 0)
92        self.assertEqual(len(result.failures), 0)
93        self.assertEqual(result.testsRun, 1)
94        self.assertEqual(result.shouldStop, False)
95
96        result.stopTest(test)
97
98        # Same tests as above; make sure nothing has changed
99        self.assertTrue(result.wasSuccessful())
100        self.assertEqual(len(result.errors), 0)
101        self.assertEqual(len(result.failures), 0)
102        self.assertEqual(result.testsRun, 1)
103        self.assertEqual(result.shouldStop, False)
104
105    # "Called before and after tests are run. The default implementation does nothing."
106    def test_startTestRun_stopTestRun(self):
107        result = unittest.TestResult()
108        result.startTestRun()
109        result.stopTestRun()
110
111    # "addSuccess(test)"
112    # ...
113    # "Called when the test case test succeeds"
114    # ...
115    # "wasSuccessful() - Returns True if all tests run so far have passed,
116    # otherwise returns False"
117    # ...
118    # "testsRun - The total number of tests run so far."
119    # ...
120    # "errors - A list containing 2-tuples of TestCase instances and
121    # formatted tracebacks. Each tuple represents a test which raised an
122    # unexpected exception. Contains formatted
123    # tracebacks instead of sys.exc_info() results."
124    # ...
125    # "failures - A list containing 2-tuples of TestCase instances and
126    # formatted tracebacks. Each tuple represents a test where a failure was
127    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
128    # methods. Contains formatted tracebacks instead
129    # of sys.exc_info() results."
130    def test_addSuccess(self):
131        class Foo(unittest.TestCase):
132            def test_1(self):
133                pass
134
135        test = Foo('test_1')
136
137        result = unittest.TestResult()
138
139        result.startTest(test)
140        result.addSuccess(test)
141        result.stopTest(test)
142
143        self.assertTrue(result.wasSuccessful())
144        self.assertEqual(len(result.errors), 0)
145        self.assertEqual(len(result.failures), 0)
146        self.assertEqual(result.testsRun, 1)
147        self.assertEqual(result.shouldStop, False)
148
149    # "addFailure(test, err)"
150    # ...
151    # "Called when the test case test signals a failure. err is a tuple of
152    # the form returned by sys.exc_info(): (type, value, traceback)"
153    # ...
154    # "wasSuccessful() - Returns True if all tests run so far have passed,
155    # otherwise returns False"
156    # ...
157    # "testsRun - The total number of tests run so far."
158    # ...
159    # "errors - A list containing 2-tuples of TestCase instances and
160    # formatted tracebacks. Each tuple represents a test which raised an
161    # unexpected exception. Contains formatted
162    # tracebacks instead of sys.exc_info() results."
163    # ...
164    # "failures - A list containing 2-tuples of TestCase instances and
165    # formatted tracebacks. Each tuple represents a test where a failure was
166    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
167    # methods. Contains formatted tracebacks instead
168    # of sys.exc_info() results."
169    def test_addFailure(self):
170        class Foo(unittest.TestCase):
171            def test_1(self):
172                pass
173
174        test = Foo('test_1')
175        try:
176            test.fail("foo")
177        except:
178            exc_info_tuple = sys.exc_info()
179
180        result = unittest.TestResult()
181
182        result.startTest(test)
183        result.addFailure(test, exc_info_tuple)
184        result.stopTest(test)
185
186        self.assertFalse(result.wasSuccessful())
187        self.assertEqual(len(result.errors), 0)
188        self.assertEqual(len(result.failures), 1)
189        self.assertEqual(result.testsRun, 1)
190        self.assertEqual(result.shouldStop, False)
191
192        test_case, formatted_exc = result.failures[0]
193        self.assertIs(test_case, test)
194        self.assertIsInstance(formatted_exc, str)
195
196    # "addError(test, err)"
197    # ...
198    # "Called when the test case test raises an unexpected exception err
199    # is a tuple of the form returned by sys.exc_info():
200    # (type, value, traceback)"
201    # ...
202    # "wasSuccessful() - Returns True if all tests run so far have passed,
203    # otherwise returns False"
204    # ...
205    # "testsRun - The total number of tests run so far."
206    # ...
207    # "errors - A list containing 2-tuples of TestCase instances and
208    # formatted tracebacks. Each tuple represents a test which raised an
209    # unexpected exception. Contains formatted
210    # tracebacks instead of sys.exc_info() results."
211    # ...
212    # "failures - A list containing 2-tuples of TestCase instances and
213    # formatted tracebacks. Each tuple represents a test where a failure was
214    # explicitly signalled using the TestCase.fail*() or TestCase.assert*()
215    # methods. Contains formatted tracebacks instead
216    # of sys.exc_info() results."
217    def test_addError(self):
218        class Foo(unittest.TestCase):
219            def test_1(self):
220                pass
221
222        test = Foo('test_1')
223        try:
224            raise TypeError()
225        except:
226            exc_info_tuple = sys.exc_info()
227
228        result = unittest.TestResult()
229
230        result.startTest(test)
231        result.addError(test, exc_info_tuple)
232        result.stopTest(test)
233
234        self.assertFalse(result.wasSuccessful())
235        self.assertEqual(len(result.errors), 1)
236        self.assertEqual(len(result.failures), 0)
237        self.assertEqual(result.testsRun, 1)
238        self.assertEqual(result.shouldStop, False)
239
240        test_case, formatted_exc = result.errors[0]
241        self.assertIs(test_case, test)
242        self.assertIsInstance(formatted_exc, str)
243
244    def test_addError_locals(self):
245        class Foo(unittest.TestCase):
246            def test_1(self):
247                1/0
248
249        test = Foo('test_1')
250        result = unittest.TestResult()
251        result.tb_locals = True
252
253        unittest.result.traceback = MockTraceback
254        self.addCleanup(restore_traceback)
255        result.startTestRun()
256        test.run(result)
257        result.stopTestRun()
258
259        self.assertEqual(len(result.errors), 1)
260        test_case, formatted_exc = result.errors[0]
261        self.assertEqual('A tracebacklocals', formatted_exc)
262
263    def test_addSubTest(self):
264        class Foo(unittest.TestCase):
265            def test_1(self):
266                nonlocal subtest
267                with self.subTest(foo=1):
268                    subtest = self._subtest
269                    try:
270                        1/0
271                    except ZeroDivisionError:
272                        exc_info_tuple = sys.exc_info()
273                    # Register an error by hand (to check the API)
274                    result.addSubTest(test, subtest, exc_info_tuple)
275                    # Now trigger a failure
276                    self.fail("some recognizable failure")
277
278        subtest = None
279        test = Foo('test_1')
280        result = unittest.TestResult()
281
282        test.run(result)
283
284        self.assertFalse(result.wasSuccessful())
285        self.assertEqual(len(result.errors), 1)
286        self.assertEqual(len(result.failures), 1)
287        self.assertEqual(result.testsRun, 1)
288        self.assertEqual(result.shouldStop, False)
289
290        test_case, formatted_exc = result.errors[0]
291        self.assertIs(test_case, subtest)
292        self.assertIn("ZeroDivisionError", formatted_exc)
293        test_case, formatted_exc = result.failures[0]
294        self.assertIs(test_case, subtest)
295        self.assertIn("some recognizable failure", formatted_exc)
296
297    def testGetDescriptionWithoutDocstring(self):
298        result = unittest.TextTestResult(None, True, 1)
299        self.assertEqual(
300                result.getDescription(self),
301                'testGetDescriptionWithoutDocstring (' + __name__ +
302                '.Test_TestResult)')
303
304    def testGetSubTestDescriptionWithoutDocstring(self):
305        with self.subTest(foo=1, bar=2):
306            result = unittest.TextTestResult(None, True, 1)
307            self.assertEqual(
308                    result.getDescription(self._subtest),
309                    'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
310                    '.Test_TestResult) (foo=1, bar=2)')
311        with self.subTest('some message'):
312            result = unittest.TextTestResult(None, True, 1)
313            self.assertEqual(
314                    result.getDescription(self._subtest),
315                    'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
316                    '.Test_TestResult) [some message]')
317
318    def testGetSubTestDescriptionWithoutDocstringAndParams(self):
319        with self.subTest():
320            result = unittest.TextTestResult(None, True, 1)
321            self.assertEqual(
322                    result.getDescription(self._subtest),
323                    'testGetSubTestDescriptionWithoutDocstringAndParams '
324                    '(' + __name__ + '.Test_TestResult) (<subtest>)')
325
326    def testGetSubTestDescriptionForFalsyValues(self):
327        expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
328        result = unittest.TextTestResult(None, True, 1)
329        for arg in [0, None, []]:
330            with self.subTest(arg):
331                self.assertEqual(
332                    result.getDescription(self._subtest),
333                    expected % (__name__, arg)
334                )
335
336    def testGetNestedSubTestDescriptionWithoutDocstring(self):
337        with self.subTest(foo=1):
338            with self.subTest(baz=2, bar=3):
339                result = unittest.TextTestResult(None, True, 1)
340                self.assertEqual(
341                        result.getDescription(self._subtest),
342                        'testGetNestedSubTestDescriptionWithoutDocstring '
343                        '(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
344
345    def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
346        with self.subTest(foo=1, bar=2):
347            with self.subTest(baz=3, bar=4):
348                result = unittest.TextTestResult(None, True, 1)
349                self.assertEqual(
350                        result.getDescription(self._subtest),
351                        'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
352                        '(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
353
354    @unittest.skipIf(sys.flags.optimize >= 2,
355                     "Docstrings are omitted with -O2 and above")
356    def testGetDescriptionWithOneLineDocstring(self):
357        """Tests getDescription() for a method with a docstring."""
358        result = unittest.TextTestResult(None, True, 1)
359        self.assertEqual(
360                result.getDescription(self),
361               ('testGetDescriptionWithOneLineDocstring '
362                '(' + __name__ + '.Test_TestResult)\n'
363                'Tests getDescription() for a method with a docstring.'))
364
365    @unittest.skipIf(sys.flags.optimize >= 2,
366                     "Docstrings are omitted with -O2 and above")
367    def testGetSubTestDescriptionWithOneLineDocstring(self):
368        """Tests getDescription() for a method with a docstring."""
369        result = unittest.TextTestResult(None, True, 1)
370        with self.subTest(foo=1, bar=2):
371            self.assertEqual(
372                result.getDescription(self._subtest),
373               ('testGetSubTestDescriptionWithOneLineDocstring '
374                '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
375                'Tests getDescription() for a method with a docstring.'))
376
377    @unittest.skipIf(sys.flags.optimize >= 2,
378                     "Docstrings are omitted with -O2 and above")
379    def testGetDescriptionWithMultiLineDocstring(self):
380        """Tests getDescription() for a method with a longer docstring.
381        The second line of the docstring.
382        """
383        result = unittest.TextTestResult(None, True, 1)
384        self.assertEqual(
385                result.getDescription(self),
386               ('testGetDescriptionWithMultiLineDocstring '
387                '(' + __name__ + '.Test_TestResult)\n'
388                'Tests getDescription() for a method with a longer '
389                'docstring.'))
390
391    @unittest.skipIf(sys.flags.optimize >= 2,
392                     "Docstrings are omitted with -O2 and above")
393    def testGetSubTestDescriptionWithMultiLineDocstring(self):
394        """Tests getDescription() for a method with a longer docstring.
395        The second line of the docstring.
396        """
397        result = unittest.TextTestResult(None, True, 1)
398        with self.subTest(foo=1, bar=2):
399            self.assertEqual(
400                result.getDescription(self._subtest),
401               ('testGetSubTestDescriptionWithMultiLineDocstring '
402                '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
403                'Tests getDescription() for a method with a longer '
404                'docstring.'))
405
406    def testStackFrameTrimming(self):
407        class Frame(object):
408            class tb_frame(object):
409                f_globals = {}
410        result = unittest.TestResult()
411        self.assertFalse(result._is_relevant_tb_level(Frame))
412
413        Frame.tb_frame.f_globals['__unittest'] = True
414        self.assertTrue(result._is_relevant_tb_level(Frame))
415
416    def testFailFast(self):
417        result = unittest.TestResult()
418        result._exc_info_to_string = lambda *_: ''
419        result.failfast = True
420        result.addError(None, None)
421        self.assertTrue(result.shouldStop)
422
423        result = unittest.TestResult()
424        result._exc_info_to_string = lambda *_: ''
425        result.failfast = True
426        result.addFailure(None, None)
427        self.assertTrue(result.shouldStop)
428
429        result = unittest.TestResult()
430        result._exc_info_to_string = lambda *_: ''
431        result.failfast = True
432        result.addUnexpectedSuccess(None)
433        self.assertTrue(result.shouldStop)
434
435    def testFailFastSetByRunner(self):
436        runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
437        def test(result):
438            self.assertTrue(result.failfast)
439        result = runner.run(test)
440
441
442classDict = dict(unittest.TestResult.__dict__)
443for m in ('addSkip', 'addExpectedFailure', 'addUnexpectedSuccess',
444           '__init__'):
445    del classDict[m]
446
447def __init__(self, stream=None, descriptions=None, verbosity=None):
448    self.failures = []
449    self.errors = []
450    self.testsRun = 0
451    self.shouldStop = False
452    self.buffer = False
453    self.tb_locals = False
454
455classDict['__init__'] = __init__
456OldResult = type('OldResult', (object,), classDict)
457
458class Test_OldTestResult(unittest.TestCase):
459
460    def assertOldResultWarning(self, test, failures):
461        with support.check_warnings(("TestResult has no add.+ method,",
462                                     RuntimeWarning)):
463            result = OldResult()
464            test.run(result)
465            self.assertEqual(len(result.failures), failures)
466
467    def testOldTestResult(self):
468        class Test(unittest.TestCase):
469            def testSkip(self):
470                self.skipTest('foobar')
471            @unittest.expectedFailure
472            def testExpectedFail(self):
473                raise TypeError
474            @unittest.expectedFailure
475            def testUnexpectedSuccess(self):
476                pass
477
478        for test_name, should_pass in (('testSkip', True),
479                                       ('testExpectedFail', True),
480                                       ('testUnexpectedSuccess', False)):
481            test = Test(test_name)
482            self.assertOldResultWarning(test, int(not should_pass))
483
484    def testOldTestTesultSetup(self):
485        class Test(unittest.TestCase):
486            def setUp(self):
487                self.skipTest('no reason')
488            def testFoo(self):
489                pass
490        self.assertOldResultWarning(Test('testFoo'), 0)
491
492    def testOldTestResultClass(self):
493        @unittest.skip('no reason')
494        class Test(unittest.TestCase):
495            def testFoo(self):
496                pass
497        self.assertOldResultWarning(Test('testFoo'), 0)
498
499    def testOldResultWithRunner(self):
500        class Test(unittest.TestCase):
501            def testFoo(self):
502                pass
503        runner = unittest.TextTestRunner(resultclass=OldResult,
504                                          stream=io.StringIO())
505        # This will raise an exception if TextTestRunner can't handle old
506        # test result objects
507        runner.run(Test('testFoo'))
508
509
510class TestOutputBuffering(unittest.TestCase):
511
512    def setUp(self):
513        self._real_out = sys.stdout
514        self._real_err = sys.stderr
515
516    def tearDown(self):
517        sys.stdout = self._real_out
518        sys.stderr = self._real_err
519
520    def testBufferOutputOff(self):
521        real_out = self._real_out
522        real_err = self._real_err
523
524        result = unittest.TestResult()
525        self.assertFalse(result.buffer)
526
527        self.assertIs(real_out, sys.stdout)
528        self.assertIs(real_err, sys.stderr)
529
530        result.startTest(self)
531
532        self.assertIs(real_out, sys.stdout)
533        self.assertIs(real_err, sys.stderr)
534
535    def testBufferOutputStartTestAddSuccess(self):
536        real_out = self._real_out
537        real_err = self._real_err
538
539        result = unittest.TestResult()
540        self.assertFalse(result.buffer)
541
542        result.buffer = True
543
544        self.assertIs(real_out, sys.stdout)
545        self.assertIs(real_err, sys.stderr)
546
547        result.startTest(self)
548
549        self.assertIsNot(real_out, sys.stdout)
550        self.assertIsNot(real_err, sys.stderr)
551        self.assertIsInstance(sys.stdout, io.StringIO)
552        self.assertIsInstance(sys.stderr, io.StringIO)
553        self.assertIsNot(sys.stdout, sys.stderr)
554
555        out_stream = sys.stdout
556        err_stream = sys.stderr
557
558        result._original_stdout = io.StringIO()
559        result._original_stderr = io.StringIO()
560
561        print('foo')
562        print('bar', file=sys.stderr)
563
564        self.assertEqual(out_stream.getvalue(), 'foo\n')
565        self.assertEqual(err_stream.getvalue(), 'bar\n')
566
567        self.assertEqual(result._original_stdout.getvalue(), '')
568        self.assertEqual(result._original_stderr.getvalue(), '')
569
570        result.addSuccess(self)
571        result.stopTest(self)
572
573        self.assertIs(sys.stdout, result._original_stdout)
574        self.assertIs(sys.stderr, result._original_stderr)
575
576        self.assertEqual(result._original_stdout.getvalue(), '')
577        self.assertEqual(result._original_stderr.getvalue(), '')
578
579        self.assertEqual(out_stream.getvalue(), '')
580        self.assertEqual(err_stream.getvalue(), '')
581
582
583    def getStartedResult(self):
584        result = unittest.TestResult()
585        result.buffer = True
586        result.startTest(self)
587        return result
588
589    def testBufferOutputAddErrorOrFailure(self):
590        unittest.result.traceback = MockTraceback
591        self.addCleanup(restore_traceback)
592
593        for message_attr, add_attr, include_error in [
594            ('errors', 'addError', True),
595            ('failures', 'addFailure', False),
596            ('errors', 'addError', True),
597            ('failures', 'addFailure', False)
598        ]:
599            result = self.getStartedResult()
600            buffered_out = sys.stdout
601            buffered_err = sys.stderr
602            result._original_stdout = io.StringIO()
603            result._original_stderr = io.StringIO()
604
605            print('foo', file=sys.stdout)
606            if include_error:
607                print('bar', file=sys.stderr)
608
609
610            addFunction = getattr(result, add_attr)
611            addFunction(self, (None, None, None))
612            result.stopTest(self)
613
614            result_list = getattr(result, message_attr)
615            self.assertEqual(len(result_list), 1)
616
617            test, message = result_list[0]
618            expectedOutMessage = textwrap.dedent("""
619                Stdout:
620                foo
621            """)
622            expectedErrMessage = ''
623            if include_error:
624                expectedErrMessage = textwrap.dedent("""
625                Stderr:
626                bar
627            """)
628
629            expectedFullMessage = 'A traceback%s%s' % (expectedOutMessage, expectedErrMessage)
630
631            self.assertIs(test, self)
632            self.assertEqual(result._original_stdout.getvalue(), expectedOutMessage)
633            self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage)
634            self.assertMultiLineEqual(message, expectedFullMessage)
635
636    def testBufferSetupClass(self):
637        result = unittest.TestResult()
638        result.buffer = True
639
640        class Foo(unittest.TestCase):
641            @classmethod
642            def setUpClass(cls):
643                1/0
644            def test_foo(self):
645                pass
646        suite = unittest.TestSuite([Foo('test_foo')])
647        suite(result)
648        self.assertEqual(len(result.errors), 1)
649
650    def testBufferTearDownClass(self):
651        result = unittest.TestResult()
652        result.buffer = True
653
654        class Foo(unittest.TestCase):
655            @classmethod
656            def tearDownClass(cls):
657                1/0
658            def test_foo(self):
659                pass
660        suite = unittest.TestSuite([Foo('test_foo')])
661        suite(result)
662        self.assertEqual(len(result.errors), 1)
663
664    def testBufferSetUpModule(self):
665        result = unittest.TestResult()
666        result.buffer = True
667
668        class Foo(unittest.TestCase):
669            def test_foo(self):
670                pass
671        class Module(object):
672            @staticmethod
673            def setUpModule():
674                1/0
675
676        Foo.__module__ = 'Module'
677        sys.modules['Module'] = Module
678        self.addCleanup(sys.modules.pop, 'Module')
679        suite = unittest.TestSuite([Foo('test_foo')])
680        suite(result)
681        self.assertEqual(len(result.errors), 1)
682
683    def testBufferTearDownModule(self):
684        result = unittest.TestResult()
685        result.buffer = True
686
687        class Foo(unittest.TestCase):
688            def test_foo(self):
689                pass
690        class Module(object):
691            @staticmethod
692            def tearDownModule():
693                1/0
694
695        Foo.__module__ = 'Module'
696        sys.modules['Module'] = Module
697        self.addCleanup(sys.modules.pop, 'Module')
698        suite = unittest.TestSuite([Foo('test_foo')])
699        suite(result)
700        self.assertEqual(len(result.errors), 1)
701
702
703if __name__ == '__main__':
704    unittest.main()
705