1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import mock 18import unittest 19 20from acts import asserts 21from acts import base_test 22from acts import signals 23from acts import test_runner 24 25MSG_EXPECTED_EXCEPTION = "This is an expected exception." 26MSG_EXPECTED_TEST_FAILURE = "This is an expected test failure." 27MSG_UNEXPECTED_EXCEPTION = "Unexpected exception!" 28 29MOCK_EXTRA = {"key": "value", "answer_to_everything": 42} 30 31 32def never_call(): 33 raise Exception(MSG_UNEXPECTED_EXCEPTION) 34 35 36class SomeError(Exception): 37 """A custom exception class used for tests in this module.""" 38 39 40class ActsBaseClassTest(unittest.TestCase): 41 def setUp(self): 42 self.mock_test_cls_configs = { 43 'reporter': mock.MagicMock(), 44 'log': mock.MagicMock(), 45 'log_path': '/tmp', 46 'cli_args': None, 47 'user_params': { 48 "some_param": "hahaha" 49 } 50 } 51 self.mock_test_name = "test_something" 52 53 def test_current_test_case_name(self): 54 class MockBaseTest(base_test.BaseTestClass): 55 def test_func(self): 56 asserts.assert_true( 57 self.current_test_name == "test_func", 58 ("Got " 59 "unexpected test name %s.") % self.current_test_name) 60 61 bt_cls = MockBaseTest(self.mock_test_cls_configs) 62 bt_cls.run(test_names=["test_func"]) 63 actual_record = bt_cls.results.passed[0] 64 self.assertEqual(actual_record.test_name, "test_func") 65 self.assertIsNone(actual_record.details) 66 self.assertIsNone(actual_record.extras) 67 68 def test_self_tests_list(self): 69 class MockBaseTest(base_test.BaseTestClass): 70 def __init__(self, controllers): 71 super(MockBaseTest, self).__init__(controllers) 72 self.tests = ("test_something", ) 73 74 def test_something(self): 75 pass 76 77 def test_never(self): 78 # This should not execute it's not on default test list. 79 never_call() 80 81 bt_cls = MockBaseTest(self.mock_test_cls_configs) 82 bt_cls.run() 83 actual_record = bt_cls.results.passed[0] 84 self.assertEqual(actual_record.test_name, "test_something") 85 86 def test_self_tests_list_fail_by_convention(self): 87 class MockBaseTest(base_test.BaseTestClass): 88 def __init__(self, controllers): 89 super(MockBaseTest, self).__init__(controllers) 90 self.tests = ("not_a_test_something", ) 91 92 def not_a_test_something(self): 93 pass 94 95 def test_never(self): 96 # This should not execute it's not on default test list. 97 never_call() 98 99 bt_cls = MockBaseTest(self.mock_test_cls_configs) 100 expected_msg = ("Test case name not_a_test_something does not follow " 101 "naming convention test_\*, abort.") 102 with self.assertRaisesRegexp(base_test.Error, expected_msg): 103 bt_cls.run() 104 105 def test_cli_test_selection_override_self_tests_list(self): 106 class MockBaseTest(base_test.BaseTestClass): 107 def __init__(self, controllers): 108 super(MockBaseTest, self).__init__(controllers) 109 self.tests = ("test_never", ) 110 111 def test_something(self): 112 pass 113 114 def test_never(self): 115 # This should not execute it's not selected by cmd line input. 116 never_call() 117 118 bt_cls = MockBaseTest(self.mock_test_cls_configs) 119 bt_cls.run(test_names=["test_something"]) 120 actual_record = bt_cls.results.passed[0] 121 self.assertEqual(actual_record.test_name, "test_something") 122 123 def test_cli_test_selection_fail_by_convention(self): 124 class MockBaseTest(base_test.BaseTestClass): 125 def __init__(self, controllers): 126 super(MockBaseTest, self).__init__(controllers) 127 self.tests = ("not_a_test_something", ) 128 129 def not_a_test_something(self): 130 pass 131 132 def test_never(self): 133 # This should not execute it's not selected by cmd line input. 134 never_call() 135 136 bt_cls = MockBaseTest(self.mock_test_cls_configs) 137 expected_msg = ("Test case name not_a_test_something does not follow " 138 "naming convention test_*, abort.") 139 with self.assertRaises(base_test.Error, msg=expected_msg): 140 bt_cls.run(test_names=["not_a_test_something"]) 141 142 def test_default_execution_of_all_tests(self): 143 class MockBaseTest(base_test.BaseTestClass): 144 def test_something(self): 145 pass 146 147 def not_a_test(self): 148 # This should not execute its name doesn't follow test case 149 # naming convention. 150 never_call() 151 152 bt_cls = MockBaseTest(self.mock_test_cls_configs) 153 bt_cls.run(test_names=["test_something"]) 154 actual_record = bt_cls.results.passed[0] 155 self.assertEqual(actual_record.test_name, "test_something") 156 157 def test_missing_requested_test_func(self): 158 class MockBaseTest(base_test.BaseTestClass): 159 pass 160 161 bt_cls = MockBaseTest(self.mock_test_cls_configs) 162 bt_cls.run(test_names=["test_something"]) 163 self.assertFalse(bt_cls.results.executed) 164 self.assertTrue(bt_cls.results.skipped) 165 166 def test_setup_class_fail_by_exception(self): 167 call_check = mock.MagicMock() 168 169 class MockBaseTest(base_test.BaseTestClass): 170 def setup_class(self): 171 raise Exception(MSG_EXPECTED_EXCEPTION) 172 173 def test_something(self): 174 # This should not execute because setup_class failed. 175 never_call() 176 177 def on_blocked(self, test_name, begin_time): 178 call_check("haha") 179 180 bt_cls = MockBaseTest(self.mock_test_cls_configs) 181 bt_cls.run() 182 actual_record = bt_cls.results.blocked[0] 183 self.assertEqual(actual_record.test_name, "test_something") 184 expected_summary = ( 185 "Blocked 1, ControllerInfo {}, Executed 0, Failed 0, Passed 0," 186 " Requested 1, Skipped 0, Unknown 0") 187 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 188 call_check.assert_called_once_with("haha") 189 190 def test_setup_test_fail_by_exception(self): 191 class MockBaseTest(base_test.BaseTestClass): 192 def setup_test(self): 193 raise Exception(MSG_EXPECTED_EXCEPTION) 194 195 def test_something(self): 196 # This should not execute because setup_test failed. 197 never_call() 198 199 bt_cls = MockBaseTest(self.mock_test_cls_configs) 200 bt_cls.run(test_names=["test_something"]) 201 actual_record = bt_cls.results.unknown[0] 202 self.assertEqual(actual_record.test_name, self.mock_test_name) 203 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 204 self.assertIsNone(actual_record.extras) 205 expected_summary = ( 206 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 207 "Requested 1, Skipped 0, Unknown 1") 208 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 209 210 def test_setup_test_fail_by_test_signal(self): 211 class MockBaseTest(base_test.BaseTestClass): 212 def setup_test(self): 213 raise signals.TestFailure(MSG_EXPECTED_EXCEPTION) 214 215 def test_something(self): 216 # This should not execute because setup_test failed. 217 never_call() 218 219 bt_cls = MockBaseTest(self.mock_test_cls_configs) 220 bt_cls.run(test_names=["test_something"]) 221 actual_record = bt_cls.results.failed[0] 222 self.assertEqual(actual_record.test_name, self.mock_test_name) 223 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 224 self.assertIsNone(actual_record.extras) 225 expected_summary = ( 226 "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " 227 "Requested 1, Skipped 0, Unknown 0") 228 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 229 230 def test_setup_test_fail_by_return_False(self): 231 class MockBaseTest(base_test.BaseTestClass): 232 def setup_test(self): 233 return False 234 235 def test_something(self): 236 # This should not execute because setup_test failed. 237 never_call() 238 239 bt_cls = MockBaseTest(self.mock_test_cls_configs) 240 bt_cls.run(test_names=["test_something"]) 241 actual_record = bt_cls.results.failed[0] 242 expected_msg = "Setup for %s failed." % self.mock_test_name 243 self.assertEqual(actual_record.test_name, self.mock_test_name) 244 self.assertEqual(actual_record.details, expected_msg) 245 self.assertIsNone(actual_record.extras) 246 expected_summary = ( 247 "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " 248 "Requested 1, Skipped 0, Unknown 0") 249 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 250 251 def test_teardown_test_assert_fail(self): 252 class MockBaseTest(base_test.BaseTestClass): 253 def teardown_test(self): 254 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 255 256 def test_something(self): 257 pass 258 259 bt_cls = MockBaseTest(self.mock_test_cls_configs) 260 bt_cls.run() 261 actual_record = bt_cls.results.unknown[0] 262 self.assertEqual(actual_record.test_name, self.mock_test_name) 263 self.assertIsNone(actual_record.details) 264 self.assertIsNone(actual_record.extras) 265 expected_summary = ( 266 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 267 "Requested 1, Skipped 0, Unknown 1") 268 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 269 270 def test_teardown_test_raise_exception(self): 271 class MockBaseTest(base_test.BaseTestClass): 272 def teardown_test(self): 273 raise Exception(MSG_EXPECTED_EXCEPTION) 274 275 def test_something(self): 276 pass 277 278 bt_cls = MockBaseTest(self.mock_test_cls_configs) 279 bt_cls.run() 280 actual_record = bt_cls.results.unknown[0] 281 self.assertEqual(actual_record.test_name, self.mock_test_name) 282 self.assertIsNone(actual_record.details) 283 self.assertIsNone(actual_record.extras) 284 expected_extra_error = {"teardown_test": MSG_EXPECTED_EXCEPTION} 285 self.assertEqual(actual_record.additional_errors, expected_extra_error) 286 expected_summary = ( 287 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 288 "Requested 1, Skipped 0, Unknown 1") 289 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 290 291 def test_teardown_test_executed_if_test_pass(self): 292 my_mock = mock.MagicMock() 293 294 class MockBaseTest(base_test.BaseTestClass): 295 def teardown_test(self): 296 my_mock("teardown_test") 297 298 def test_something(self): 299 pass 300 301 bt_cls = MockBaseTest(self.mock_test_cls_configs) 302 bt_cls.run() 303 actual_record = bt_cls.results.passed[0] 304 my_mock.assert_called_once_with("teardown_test") 305 self.assertEqual(actual_record.test_name, self.mock_test_name) 306 self.assertIsNone(actual_record.details) 307 self.assertIsNone(actual_record.extras) 308 expected_summary = ( 309 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 1, " 310 "Requested 1, Skipped 0, Unknown 0") 311 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 312 313 def test_teardown_test_executed_if_setup_test_fails(self): 314 my_mock = mock.MagicMock() 315 316 class MockBaseTest(base_test.BaseTestClass): 317 def setup_test(self): 318 raise Exception(MSG_EXPECTED_EXCEPTION) 319 320 def teardown_test(self): 321 my_mock("teardown_test") 322 323 def test_something(self): 324 pass 325 326 bt_cls = MockBaseTest(self.mock_test_cls_configs) 327 bt_cls.run() 328 actual_record = bt_cls.results.unknown[0] 329 my_mock.assert_called_once_with("teardown_test") 330 self.assertEqual(actual_record.test_name, self.mock_test_name) 331 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 332 self.assertIsNone(actual_record.extras) 333 expected_summary = ( 334 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 335 "Requested 1, Skipped 0, Unknown 1") 336 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 337 338 def test_teardown_test_executed_if_test_fails(self): 339 my_mock = mock.MagicMock() 340 341 class MockBaseTest(base_test.BaseTestClass): 342 def teardown_test(self): 343 my_mock("teardown_test") 344 345 def test_something(self): 346 raise Exception(MSG_EXPECTED_EXCEPTION) 347 348 bt_cls = MockBaseTest(self.mock_test_cls_configs) 349 bt_cls.run() 350 actual_record = bt_cls.results.unknown[0] 351 my_mock.assert_called_once_with("teardown_test") 352 self.assertEqual(actual_record.test_name, self.mock_test_name) 353 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 354 self.assertIsNone(actual_record.extras) 355 expected_summary = ( 356 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 357 "Requested 1, Skipped 0, Unknown 1") 358 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 359 360 def test_on_exception_executed_if_teardown_test_fails(self): 361 my_mock = mock.MagicMock() 362 363 class MockBaseTest(base_test.BaseTestClass): 364 def on_exception(self, test_name, begin_time): 365 my_mock("on_exception") 366 367 def teardown_test(self): 368 raise Exception(MSG_EXPECTED_EXCEPTION) 369 370 def test_something(self): 371 pass 372 373 bt_cls = MockBaseTest(self.mock_test_cls_configs) 374 bt_cls.run() 375 my_mock.assert_called_once_with("on_exception") 376 actual_record = bt_cls.results.unknown[0] 377 self.assertEqual(actual_record.test_name, self.mock_test_name) 378 self.assertIsNone(actual_record.details) 379 self.assertIsNone(actual_record.extras) 380 expected_summary = ( 381 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 382 "Requested 1, Skipped 0, Unknown 1") 383 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 384 385 def test_on_fail_executed_if_test_fails(self): 386 my_mock = mock.MagicMock() 387 388 class MockBaseTest(base_test.BaseTestClass): 389 def on_fail(self, test_name, begin_time): 390 my_mock("on_fail") 391 392 def test_something(self): 393 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 394 395 bt_cls = MockBaseTest(self.mock_test_cls_configs) 396 bt_cls.run() 397 my_mock.assert_called_once_with("on_fail") 398 actual_record = bt_cls.results.failed[0] 399 self.assertEqual(actual_record.test_name, self.mock_test_name) 400 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 401 self.assertIsNone(actual_record.extras) 402 expected_summary = ( 403 "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " 404 "Requested 1, Skipped 0, Unknown 0") 405 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 406 407 def test_on_fail_executed_if_test_setup_fails_by_exception(self): 408 my_mock = mock.MagicMock() 409 410 class MockBaseTest(base_test.BaseTestClass): 411 def setup_test(self): 412 raise Exception(MSG_EXPECTED_EXCEPTION) 413 414 def on_fail(self, test_name, begin_time): 415 my_mock("on_fail") 416 417 def test_something(self): 418 pass 419 420 bt_cls = MockBaseTest(self.mock_test_cls_configs) 421 bt_cls.run() 422 my_mock.assert_called_once_with("on_fail") 423 actual_record = bt_cls.results.unknown[0] 424 self.assertEqual(actual_record.test_name, self.mock_test_name) 425 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 426 self.assertIsNone(actual_record.extras) 427 expected_summary = ( 428 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 429 "Requested 1, Skipped 0, Unknown 1") 430 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 431 432 def test_on_fail_executed_if_test_setup_fails_by_return_False(self): 433 my_mock = mock.MagicMock() 434 435 class MockBaseTest(base_test.BaseTestClass): 436 def setup_test(self): 437 return False 438 439 def on_fail(self, test_name, begin_time): 440 my_mock("on_fail") 441 442 def test_something(self): 443 pass 444 445 bt_cls = MockBaseTest(self.mock_test_cls_configs) 446 bt_cls.run() 447 my_mock.assert_called_once_with("on_fail") 448 actual_record = bt_cls.results.failed[0] 449 self.assertEqual(actual_record.test_name, self.mock_test_name) 450 self.assertEqual(actual_record.details, 451 'Setup for test_something failed.') 452 self.assertIsNone(actual_record.extras) 453 expected_summary = ( 454 "Blocked 0, ControllerInfo {}, Executed 1, Failed 1, Passed 0, " 455 "Requested 1, Skipped 0, Unknown 0") 456 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 457 458 def test_failure_to_call_procedure_function_is_recorded(self): 459 class MockBaseTest(base_test.BaseTestClass): 460 def on_fail(self): 461 pass 462 463 def test_something(self): 464 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 465 466 bt_cls = MockBaseTest(self.mock_test_cls_configs) 467 bt_cls.run() 468 actual_record = bt_cls.results.unknown[0] 469 self.assertIn('_on_fail', actual_record.additional_errors) 470 self.assertEqual(actual_record.test_name, self.mock_test_name) 471 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 472 self.assertIsNone(actual_record.extras) 473 expected_summary = ( 474 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 475 "Requested 1, Skipped 0, Unknown 1") 476 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 477 478 def test_failure_in_procedure_functions_is_recorded(self): 479 expected_msg = "Something failed in on_pass." 480 481 class MockBaseTest(base_test.BaseTestClass): 482 def on_pass(self, test_name, begin_time): 483 raise Exception(expected_msg) 484 485 def test_something(self): 486 asserts.explicit_pass(MSG_EXPECTED_EXCEPTION) 487 488 bt_cls = MockBaseTest(self.mock_test_cls_configs) 489 bt_cls.run() 490 actual_record = bt_cls.results.unknown[0] 491 expected_extra_error = {'_on_pass': expected_msg} 492 self.assertEqual(actual_record.additional_errors, expected_extra_error) 493 self.assertEqual(actual_record.test_name, self.mock_test_name) 494 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 495 self.assertIsNone(actual_record.extras) 496 expected_summary = ( 497 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 498 "Requested 1, Skipped 0, Unknown 1") 499 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 500 501 def test_both_teardown_and_test_body_raise_exceptions(self): 502 class MockBaseTest(base_test.BaseTestClass): 503 def teardown_test(self): 504 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 505 506 def test_something(self): 507 raise Exception("Test Body Exception.") 508 509 bt_cls = MockBaseTest(self.mock_test_cls_configs) 510 bt_cls.run() 511 actual_record = bt_cls.results.unknown[0] 512 self.assertEqual(actual_record.test_name, self.mock_test_name) 513 self.assertEqual(actual_record.details, "Test Body Exception.") 514 self.assertIsNone(actual_record.extras) 515 self.assertEqual(actual_record.additional_errors["teardown_test"], 516 "Details=This is an expected exception., Extras=None") 517 expected_summary = ( 518 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 519 "Requested 1, Skipped 0, Unknown 1") 520 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 521 522 def test_explicit_pass_but_teardown_test_raises_an_exception(self): 523 """Test record result should be marked as UNKNOWN as opposed to PASS. 524 """ 525 526 class MockBaseTest(base_test.BaseTestClass): 527 def teardown_test(self): 528 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION) 529 530 def test_something(self): 531 asserts.explicit_pass("Test Passed!") 532 533 bt_cls = MockBaseTest(self.mock_test_cls_configs) 534 bt_cls.run() 535 actual_record = bt_cls.results.unknown[0] 536 self.assertEqual(actual_record.test_name, self.mock_test_name) 537 self.assertEqual(actual_record.details, "Test Passed!") 538 self.assertIsNone(actual_record.extras) 539 self.assertEqual(actual_record.additional_errors["teardown_test"], 540 "Details=This is an expected exception., Extras=None") 541 expected_summary = ( 542 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 543 "Requested 1, Skipped 0, Unknown 1") 544 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 545 546 def test_on_pass_raise_exception(self): 547 class MockBaseTest(base_test.BaseTestClass): 548 def on_pass(self, test_name, begin_time): 549 raise Exception(MSG_EXPECTED_EXCEPTION) 550 551 def test_something(self): 552 asserts.explicit_pass( 553 MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 554 555 bt_cls = MockBaseTest(self.mock_test_cls_configs) 556 bt_cls.run() 557 actual_record = bt_cls.results.unknown[0] 558 self.assertEqual(actual_record.test_name, self.mock_test_name) 559 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 560 self.assertEqual(actual_record.extras, MOCK_EXTRA) 561 self.assertEqual(actual_record.additional_errors, { 562 '_on_pass': MSG_EXPECTED_EXCEPTION 563 }) 564 expected_summary = ( 565 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 566 "Requested 1, Skipped 0, Unknown 1") 567 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 568 569 def test_on_fail_raise_exception(self): 570 class MockBaseTest(base_test.BaseTestClass): 571 def on_fail(self, test_name, begin_time): 572 raise Exception(MSG_EXPECTED_EXCEPTION) 573 574 def test_something(self): 575 asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 576 577 bt_cls = MockBaseTest(self.mock_test_cls_configs) 578 bt_cls.run() 579 actual_record = bt_cls.results.unknown[0] 580 self.assertEqual(bt_cls.results.failed, []) 581 self.assertEqual(actual_record.test_name, self.mock_test_name) 582 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 583 self.assertEqual(actual_record.extras, MOCK_EXTRA) 584 self.assertEqual(actual_record.additional_errors, { 585 '_on_fail': MSG_EXPECTED_EXCEPTION 586 }) 587 expected_summary = ( 588 "Blocked 0, ControllerInfo {}, Executed 1, Failed 0, Passed 0, " 589 "Requested 1, Skipped 0, Unknown 1") 590 self.assertEqual(bt_cls.results.summary_str(), expected_summary) 591 592 def test_abort_class(self): 593 class MockBaseTest(base_test.BaseTestClass): 594 def test_1(self): 595 pass 596 597 def test_2(self): 598 asserts.abort_class(MSG_EXPECTED_EXCEPTION) 599 never_call() 600 601 def test_3(self): 602 never_call() 603 604 bt_cls = MockBaseTest(self.mock_test_cls_configs) 605 bt_cls.run(test_names=["test_1", "test_2", "test_3"]) 606 self.assertEqual(bt_cls.results.passed[0].test_name, "test_1") 607 self.assertEqual(bt_cls.results.failed[0].details, 608 MSG_EXPECTED_EXCEPTION) 609 self.assertEqual( 610 bt_cls.results.summary_str(), 611 ("Blocked 0, ControllerInfo {}, Executed 2, Failed 1, Passed 1, " 612 "Requested 3, Skipped 0, Unknown 0")) 613 614 def test_uncaught_exception(self): 615 class MockBaseTest(base_test.BaseTestClass): 616 def test_func(self): 617 raise Exception(MSG_EXPECTED_EXCEPTION) 618 never_call() 619 620 bt_cls = MockBaseTest(self.mock_test_cls_configs) 621 bt_cls.run(test_names=["test_func"]) 622 actual_record = bt_cls.results.unknown[0] 623 self.assertEqual(actual_record.test_name, "test_func") 624 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 625 self.assertIsNone(actual_record.extras) 626 627 def test_fail(self): 628 class MockBaseTest(base_test.BaseTestClass): 629 def test_func(self): 630 asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 631 never_call() 632 633 bt_cls = MockBaseTest(self.mock_test_cls_configs) 634 bt_cls.run(test_names=["test_func"]) 635 actual_record = bt_cls.results.failed[0] 636 self.assertEqual(actual_record.test_name, "test_func") 637 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 638 self.assertEqual(actual_record.extras, MOCK_EXTRA) 639 640 def test_assert_true(self): 641 class MockBaseTest(base_test.BaseTestClass): 642 def test_func(self): 643 asserts.assert_true( 644 False, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 645 never_call() 646 647 bt_cls = MockBaseTest(self.mock_test_cls_configs) 648 bt_cls.run(test_names=["test_func"]) 649 actual_record = bt_cls.results.failed[0] 650 self.assertEqual(actual_record.test_name, "test_func") 651 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 652 self.assertEqual(actual_record.extras, MOCK_EXTRA) 653 654 def test_assert_equal_pass(self): 655 class MockBaseTest(base_test.BaseTestClass): 656 def test_func(self): 657 asserts.assert_equal(1, 1, extras=MOCK_EXTRA) 658 659 bt_cls = MockBaseTest(self.mock_test_cls_configs) 660 bt_cls.run() 661 actual_record = bt_cls.results.passed[0] 662 self.assertEqual(actual_record.test_name, "test_func") 663 self.assertIsNone(actual_record.details) 664 self.assertIsNone(actual_record.extras) 665 666 def test_assert_equal_fail(self): 667 class MockBaseTest(base_test.BaseTestClass): 668 def test_func(self): 669 asserts.assert_equal(1, 2, extras=MOCK_EXTRA) 670 671 bt_cls = MockBaseTest(self.mock_test_cls_configs) 672 bt_cls.run() 673 actual_record = bt_cls.results.failed[0] 674 self.assertEqual(actual_record.test_name, "test_func") 675 self.assertEqual(actual_record.details, "1 != 2") 676 self.assertEqual(actual_record.extras, MOCK_EXTRA) 677 678 def test_assert_equal_fail_with_msg(self): 679 class MockBaseTest(base_test.BaseTestClass): 680 def test_func(self): 681 asserts.assert_equal( 682 1, 2, msg=MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 683 684 bt_cls = MockBaseTest(self.mock_test_cls_configs) 685 bt_cls.run() 686 actual_record = bt_cls.results.failed[0] 687 self.assertEqual(actual_record.test_name, "test_func") 688 expected_msg = "1 != 2 " + MSG_EXPECTED_EXCEPTION 689 self.assertEqual(actual_record.details, expected_msg) 690 self.assertEqual(actual_record.extras, MOCK_EXTRA) 691 692 def test_assert_raises_pass(self): 693 class MockBaseTest(base_test.BaseTestClass): 694 def test_func(self): 695 with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): 696 raise SomeError(MSG_EXPECTED_EXCEPTION) 697 698 bt_cls = MockBaseTest(self.mock_test_cls_configs) 699 bt_cls.run() 700 actual_record = bt_cls.results.passed[0] 701 self.assertEqual(actual_record.test_name, "test_func") 702 self.assertIsNone(actual_record.details) 703 self.assertIsNone(actual_record.extras) 704 705 def test_assert_raises_fail_with_noop(self): 706 class MockBaseTest(base_test.BaseTestClass): 707 def test_func(self): 708 with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): 709 pass 710 711 bt_cls = MockBaseTest(self.mock_test_cls_configs) 712 bt_cls.run() 713 actual_record = bt_cls.results.failed[0] 714 self.assertEqual(actual_record.test_name, "test_func") 715 self.assertEqual(actual_record.details, "SomeError not raised") 716 self.assertEqual(actual_record.extras, MOCK_EXTRA) 717 718 def test_assert_raises_fail_with_wrong_error(self): 719 class MockBaseTest(base_test.BaseTestClass): 720 def test_func(self): 721 with asserts.assert_raises(SomeError, extras=MOCK_EXTRA): 722 raise AttributeError(MSG_UNEXPECTED_EXCEPTION) 723 724 bt_cls = MockBaseTest(self.mock_test_cls_configs) 725 bt_cls.run() 726 actual_record = bt_cls.results.unknown[0] 727 self.assertEqual(actual_record.test_name, "test_func") 728 self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION) 729 self.assertIsNone(actual_record.extras) 730 731 def test_assert_raises_regex_pass(self): 732 class MockBaseTest(base_test.BaseTestClass): 733 def test_func(self): 734 with asserts.assert_raises_regex( 735 SomeError, 736 expected_regex=MSG_EXPECTED_EXCEPTION, 737 extras=MOCK_EXTRA): 738 raise SomeError(MSG_EXPECTED_EXCEPTION) 739 740 bt_cls = MockBaseTest(self.mock_test_cls_configs) 741 bt_cls.run() 742 actual_record = bt_cls.results.passed[0] 743 self.assertEqual(actual_record.test_name, "test_func") 744 self.assertIsNone(actual_record.details) 745 self.assertIsNone(actual_record.extras) 746 747 def test_assert_raises_fail_with_noop(self): 748 class MockBaseTest(base_test.BaseTestClass): 749 def test_func(self): 750 with asserts.assert_raises_regex( 751 SomeError, 752 expected_regex=MSG_EXPECTED_EXCEPTION, 753 extras=MOCK_EXTRA): 754 pass 755 756 bt_cls = MockBaseTest(self.mock_test_cls_configs) 757 bt_cls.run() 758 actual_record = bt_cls.results.failed[0] 759 self.assertEqual(actual_record.test_name, "test_func") 760 self.assertEqual(actual_record.details, "SomeError not raised") 761 self.assertEqual(actual_record.extras, MOCK_EXTRA) 762 763 def test_assert_raises_fail_with_wrong_regex(self): 764 wrong_msg = "ha" 765 766 class MockBaseTest(base_test.BaseTestClass): 767 def test_func(self): 768 with asserts.assert_raises_regex( 769 SomeError, 770 expected_regex=MSG_EXPECTED_EXCEPTION, 771 extras=MOCK_EXTRA): 772 raise SomeError(wrong_msg) 773 774 bt_cls = MockBaseTest(self.mock_test_cls_configs) 775 bt_cls.run() 776 actual_record = bt_cls.results.failed[0] 777 self.assertEqual(actual_record.test_name, "test_func") 778 expected_details = ('"This is an expected exception." does not match ' 779 '"%s"') % wrong_msg 780 self.assertEqual(actual_record.details, expected_details) 781 self.assertEqual(actual_record.extras, MOCK_EXTRA) 782 783 def test_assert_raises_fail_with_wrong_error(self): 784 class MockBaseTest(base_test.BaseTestClass): 785 def test_func(self): 786 with asserts.assert_raises_regex( 787 SomeError, 788 expected_regex=MSG_EXPECTED_EXCEPTION, 789 extras=MOCK_EXTRA): 790 raise AttributeError(MSG_UNEXPECTED_EXCEPTION) 791 792 bt_cls = MockBaseTest(self.mock_test_cls_configs) 793 bt_cls.run() 794 actual_record = bt_cls.results.unknown[0] 795 self.assertEqual(actual_record.test_name, "test_func") 796 self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION) 797 self.assertIsNone(actual_record.extras) 798 799 def test_explicit_pass(self): 800 class MockBaseTest(base_test.BaseTestClass): 801 def test_func(self): 802 asserts.explicit_pass( 803 MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 804 never_call() 805 806 bt_cls = MockBaseTest(self.mock_test_cls_configs) 807 bt_cls.run(test_names=["test_func"]) 808 actual_record = bt_cls.results.passed[0] 809 self.assertEqual(actual_record.test_name, "test_func") 810 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 811 self.assertEqual(actual_record.extras, MOCK_EXTRA) 812 813 def test_implicit_pass(self): 814 class MockBaseTest(base_test.BaseTestClass): 815 def test_func(self): 816 pass 817 818 bt_cls = MockBaseTest(self.mock_test_cls_configs) 819 bt_cls.run(test_names=["test_func"]) 820 actual_record = bt_cls.results.passed[0] 821 self.assertEqual(actual_record.test_name, "test_func") 822 self.assertIsNone(actual_record.details) 823 self.assertIsNone(actual_record.extras) 824 825 def test_skip(self): 826 class MockBaseTest(base_test.BaseTestClass): 827 def test_func(self): 828 asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 829 never_call() 830 831 bt_cls = MockBaseTest(self.mock_test_cls_configs) 832 bt_cls.run(test_names=["test_func"]) 833 actual_record = bt_cls.results.skipped[0] 834 self.assertEqual(actual_record.test_name, "test_func") 835 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 836 self.assertEqual(actual_record.extras, MOCK_EXTRA) 837 838 def test_skip_if(self): 839 class MockBaseTest(base_test.BaseTestClass): 840 def test_func(self): 841 asserts.skip_if(False, MSG_UNEXPECTED_EXCEPTION) 842 asserts.skip_if( 843 True, MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 844 never_call() 845 846 bt_cls = MockBaseTest(self.mock_test_cls_configs) 847 bt_cls.run(test_names=["test_func"]) 848 actual_record = bt_cls.results.skipped[0] 849 self.assertEqual(actual_record.test_name, "test_func") 850 self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION) 851 self.assertEqual(actual_record.extras, MOCK_EXTRA) 852 853 def test_unpack_userparams_required(self): 854 """Missing a required param should raise an error.""" 855 required = ["some_param"] 856 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 857 bc.unpack_userparams(required) 858 expected_value = self.mock_test_cls_configs["user_params"][ 859 "some_param"] 860 self.assertEqual(bc.some_param, expected_value) 861 862 def test_unpack_userparams_required_missing(self): 863 """Missing a required param should raise an error.""" 864 required = ["something"] 865 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 866 expected_msg = ("Missing required user param '%s' in test " 867 "configuration.") % required[0] 868 with self.assertRaises(base_test.Error, msg=expected_msg): 869 bc.unpack_userparams(required) 870 871 def test_unpack_userparams_optional(self): 872 """If an optional param is specified, the value should be what's in the 873 config. 874 """ 875 opt = ["some_param"] 876 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 877 bc.unpack_userparams(opt_param_names=opt) 878 expected_value = self.mock_test_cls_configs["user_params"][ 879 "some_param"] 880 self.assertEqual(bc.some_param, expected_value) 881 882 def test_unpack_userparams_optional_with_default(self): 883 """If an optional param is specified with a default value, and the 884 param is not in the config, the value should be the default value. 885 """ 886 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 887 bc.unpack_userparams(optional_thing="whatever") 888 self.assertEqual(bc.optional_thing, "whatever") 889 890 def test_unpack_userparams_default_overwrite_by_optional_param_list(self): 891 """If an optional param is specified in kwargs, and the param is in the 892 config, the value should be the one in the config. 893 """ 894 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 895 bc.unpack_userparams(some_param="whatever") 896 expected_value = self.mock_test_cls_configs["user_params"][ 897 "some_param"] 898 self.assertEqual(bc.some_param, expected_value) 899 900 def test_unpack_userparams_default_overwrite_by_required_param_list(self): 901 """If an optional param is specified in kwargs, the param is in the 902 required param list, and the param is not specified in the config, the 903 param's alue should be the default value and there should be no error 904 thrown. 905 """ 906 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 907 bc.unpack_userparams( 908 req_param_names=['a_kwarg_param'], a_kwarg_param="whatever") 909 self.assertEqual(bc.a_kwarg_param, "whatever") 910 911 def test_unpack_userparams_optional_missing(self): 912 """Missing an optional param should not raise an error.""" 913 opt = ["something"] 914 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 915 bc.unpack_userparams(opt_param_names=opt) 916 917 def test_unpack_userparams_basic(self): 918 """Required and optional params are unpacked properly.""" 919 required = ["something"] 920 optional = ["something_else"] 921 configs = dict(self.mock_test_cls_configs) 922 configs["user_params"]["something"] = 42 923 configs["user_params"]["something_else"] = 53 924 bc = base_test.BaseTestClass(configs) 925 bc.unpack_userparams( 926 req_param_names=required, opt_param_names=optional) 927 self.assertEqual(bc.something, 42) 928 self.assertEqual(bc.something_else, 53) 929 930 def test_unpack_userparams_default_overwrite(self): 931 default_arg_val = "haha" 932 actual_arg_val = "wawa" 933 arg_name = "arg1" 934 configs = dict(self.mock_test_cls_configs) 935 configs["user_params"][arg_name] = actual_arg_val 936 bc = base_test.BaseTestClass(configs) 937 bc.unpack_userparams(opt_param_names=[arg_name], arg1=default_arg_val) 938 self.assertEqual(bc.arg1, actual_arg_val) 939 940 def test_unpack_userparams_default_None(self): 941 bc = base_test.BaseTestClass(self.mock_test_cls_configs) 942 bc.unpack_userparams(arg1="haha") 943 self.assertEqual(bc.arg1, "haha") 944 945 def test_generated_tests(self): 946 """Execute code paths for generated test cases. 947 948 Three test cases are generated, each of them produces a different 949 result: one pass, one fail, and one skip. 950 951 This test verifies that the exact three tests are executed and their 952 results are reported correctly. 953 """ 954 static_arg = "haha" 955 static_kwarg = "meh" 956 itrs = ["pass", "fail", "skip"] 957 958 class MockBaseTest(base_test.BaseTestClass): 959 def name_gen(self, setting, arg, special_arg=None): 960 return "test_%s_%s" % (setting, arg) 961 962 def logic(self, setting, arg, special_arg=None): 963 asserts.assert_true( 964 setting in itrs, 965 ("%s is not in acceptable settings range %s") % (setting, 966 itrs)) 967 asserts.assert_true(arg == static_arg, 968 "Expected %s, got %s" % (static_arg, arg)) 969 asserts.assert_true(arg == static_arg, "Expected %s, got %s" % 970 (static_kwarg, special_arg)) 971 if setting == "pass": 972 asserts.explicit_pass( 973 MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 974 elif setting == "fail": 975 asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 976 elif setting == "skip": 977 asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA) 978 979 @signals.generated_test 980 def test_func(self): 981 self.run_generated_testcases( 982 test_func=self.logic, 983 settings=itrs, 984 args=(static_arg, ), 985 name_func=self.name_gen) 986 987 bt_cls = MockBaseTest(self.mock_test_cls_configs) 988 bt_cls.run(test_names=["test_func"]) 989 self.assertEqual(len(bt_cls.results.requested), 3) 990 pass_record = bt_cls.results.passed[0] 991 self.assertEqual(pass_record.test_name, "test_pass_%s" % static_arg) 992 self.assertEqual(pass_record.details, MSG_EXPECTED_EXCEPTION) 993 self.assertEqual(pass_record.extras, MOCK_EXTRA) 994 skip_record = bt_cls.results.skipped[0] 995 self.assertEqual(skip_record.test_name, "test_skip_%s" % static_arg) 996 self.assertEqual(skip_record.details, MSG_EXPECTED_EXCEPTION) 997 self.assertEqual(skip_record.extras, MOCK_EXTRA) 998 fail_record = bt_cls.results.failed[0] 999 self.assertEqual(fail_record.test_name, "test_fail_%s" % static_arg) 1000 self.assertEqual(fail_record.details, MSG_EXPECTED_EXCEPTION) 1001 self.assertEqual(fail_record.extras, MOCK_EXTRA) 1002 1003 1004if __name__ == "__main__": 1005 unittest.main() 1006