1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Unit tests for the `repair` module.""" 6 7# pylint: disable=missing-docstring 8 9import functools 10import logging 11import unittest 12 13import common 14from autotest_lib.client.common_lib import hosts 15from autotest_lib.client.common_lib.hosts import repair 16from autotest_lib.server import constants 17from autotest_lib.server.hosts import host_info 18 19 20class _StubHost(object): 21 """ 22 Stub class to fill in the relevant methods of `Host`. 23 24 This class provides mocking and stub behaviors for `Host` for use by 25 tests within this module. The class implements only those methods 26 that `Verifier` and `RepairAction` actually use. 27 """ 28 29 def __init__(self): 30 self._record_sequence = [] 31 fake_board_name = constants.Labels.BOARD_PREFIX + 'fubar' 32 info = host_info.HostInfo(labels=[fake_board_name]) 33 self.host_info_store = host_info.InMemoryHostInfoStore(info) 34 self.hostname = 'unittest_host' 35 36 37 def record(self, status_code, subdir, operation, status=''): 38 """ 39 Mock method to capture records written to `status.log`. 40 41 Each record is remembered in order to be checked for correctness 42 by individual tests later. 43 44 @param status_code As for `Host.record()`. 45 @param subdir As for `Host.record()`. 46 @param operation As for `Host.record()`. 47 @param status As for `Host.record()`. 48 """ 49 full_record = (status_code, subdir, operation, status) 50 self._record_sequence.append(full_record) 51 52 53 def get_log_records(self): 54 """ 55 Return the records logged for this fake host. 56 57 The returned list of records excludes records where the 58 `operation` parameter is not in `tagset`. 59 60 @param tagset Only include log records with these tags. 61 """ 62 return self._record_sequence 63 64 65 def reset_log_records(self): 66 """Clear our history of log records to allow re-testing.""" 67 self._record_sequence = [] 68 69 70class _StubVerifier(hosts.Verifier): 71 """ 72 Stub implementation of `Verifier` for testing purposes. 73 74 This is a full implementation of a concrete `Verifier` subclass 75 designed to allow calling unit tests control over whether verify 76 passes or fails. 77 78 A `_StubVerifier()` will pass whenever the value of `_fail_count` 79 is non-zero. Calls to `try_repair()` (typically made by a 80 `_StubRepairAction()`) will reduce this count, eventually 81 "repairing" the verifier. 82 83 @property verify_count The number of calls made to the instance's 84 `verify()` method. 85 @property message If verification fails, the exception raised, 86 when converted to a string, will have this 87 value. 88 @property _fail_count The number of repair attempts required 89 before this verifier will succeed. A 90 non-zero value means verification will fail. 91 @property _description The value of the `description` property. 92 """ 93 94 def __init__(self, tag, deps, fail_count): 95 super(_StubVerifier, self).__init__(tag, deps) 96 self.verify_count = 0 97 self.message = 'Failing "%s" by request' % tag 98 self._fail_count = fail_count 99 self._description = 'Testing verify() for "%s"' % tag 100 self._log_record_map = { 101 r[0]: r for r in [ 102 ('GOOD', None, self._record_tag, ''), 103 ('FAIL', None, self._record_tag, self.message), 104 ] 105 } 106 107 108 def __repr__(self): 109 return '_StubVerifier(%r, %r, %r)' % ( 110 self.tag, self._dependency_list, self._fail_count) 111 112 113 def verify(self, host): 114 self.verify_count += 1 115 if self._fail_count: 116 raise hosts.AutoservVerifyError(self.message) 117 118 119 def try_repair(self): 120 """Bring ourselves one step closer to working.""" 121 if self._fail_count: 122 self._fail_count -= 1 123 124 125 def unrepair(self): 126 """Make ourselves more broken.""" 127 self._fail_count += 1 128 129 130 def get_log_record(self, status): 131 """ 132 Return a host log record for this verifier. 133 134 Calculates the arguments expected to be passed to 135 `Host.record()` by `Verifier._verify_host()` when this verifier 136 runs. The passed in `status` corresponds to the argument of the 137 same name to be passed to `Host.record()`. 138 139 @param status Status value of the log record. 140 """ 141 return self._log_record_map[status] 142 143 144 @property 145 def description(self): 146 return self._description 147 148 149class _StubRepairFailure(Exception): 150 """Exception to be raised by `_StubRepairAction.repair()`.""" 151 pass 152 153 154class _StubRepairAction(hosts.RepairAction): 155 """Stub implementation of `RepairAction` for testing purposes. 156 157 This is a full implementation of a concrete `RepairAction` subclass 158 designed to allow calling unit tests control over whether repair 159 passes or fails. 160 161 The behavior of `repair()` depends on the `_success` property of a 162 `_StubRepairAction`. When the property is true, repair will call 163 `try_repair()` for all triggers, and then report success. When the 164 property is false, repair reports failure. 165 166 @property repair_count The number of calls made to the instance's 167 `repair()` method. 168 @property message If repair fails, the exception raised, when 169 converted to a string, will have this value. 170 @property _success Whether repair will follow its "success" or 171 "failure" paths. 172 @property _description The value of the `description` property. 173 """ 174 175 def __init__(self, tag, deps, triggers, host_class, success): 176 super(_StubRepairAction, self).__init__(tag, deps, triggers, 177 host_class) 178 self.repair_count = 0 179 self.message = 'Failed repair for "%s"' % tag 180 self._success = success 181 self._description = 'Testing repair for "%s"' % tag 182 self._log_record_map = { 183 r[0]: r for r in [ 184 ('START', None, self._record_tag, ''), 185 ('FAIL', None, self._record_tag, self.message), 186 ('END FAIL', None, self._record_tag, ''), 187 ('END GOOD', None, self._record_tag, ''), 188 ] 189 } 190 191 192 def __repr__(self): 193 return '_StubRepairAction(%r, %r, %r, %r)' % ( 194 self.tag, self._dependency_list, 195 self._trigger_list, self._success) 196 197 198 def repair(self, host): 199 self.repair_count += 1 200 if not self._success: 201 raise _StubRepairFailure(self.message) 202 for v in self._trigger_list: 203 v.try_repair() 204 205 206 def get_log_record(self, status): 207 """ 208 Return a host log record for this repair action. 209 210 Calculates the arguments expected to be passed to 211 `Host.record()` by `RepairAction._repair_host()` when repair 212 runs. The passed in `status` corresponds to the argument of the 213 same name to be passed to `Host.record()`. 214 215 @param status Status value of the log record. 216 """ 217 return self._log_record_map[status] 218 219 220 @property 221 def description(self): 222 return self._description 223 224 225class _DependencyNodeTestCase(unittest.TestCase): 226 """ 227 Abstract base class for `RepairAction` and `Verifier` test cases. 228 229 This class provides `_make_verifier()` and `_make_repair_action()` 230 methods to create `_StubVerifier` and `_StubRepairAction` instances, 231 respectively, for testing. Constructed verifiers and repair actions 232 are remembered in `self.nodes`, a dictionary indexed by the tag 233 used to construct the object. 234 """ 235 236 def setUp(self): 237 logging.disable(logging.CRITICAL) 238 self._fake_host = _StubHost() 239 self.nodes = {} 240 241 242 def tearDown(self): 243 logging.disable(logging.NOTSET) 244 245 246 def _make_verifier(self, count, tag, deps): 247 """ 248 Make a `_StubVerifier` and remember it in `self.nodes`. 249 250 @param count As for the `_StubVerifer` constructor. 251 @param tag As for the `_StubVerifer` constructor. 252 @param deps As for the `_StubVerifer` constructor. 253 """ 254 verifier = _StubVerifier(tag, deps, count) 255 self.nodes[tag] = verifier 256 return verifier 257 258 259 def _make_repair_action(self, success, tag, deps, triggers, 260 host_class='unittest'): 261 """ 262 Make a `_StubRepairAction` and remember it in `self.nodes`. 263 264 @param success As for the `_StubRepairAction` constructor. 265 @param tag As for the `_StubRepairAction` constructor. 266 @param deps As for the `_StubRepairAction` constructor. 267 @param triggers As for the `_StubRepairAction` constructor. 268 @param host_class As for the `_StubRepairAction` constructor. 269 """ 270 repair_action = _StubRepairAction(tag, deps, triggers, host_class, 271 success) 272 self.nodes[tag] = repair_action 273 return repair_action 274 275 276 def _make_expected_failures(self, *verifiers): 277 """ 278 Make a set of `_DependencyFailure` objects from `verifiers`. 279 280 Return the set of `_DependencyFailure` objects that we would 281 expect to see in the `failures` attribute of an 282 `AutoservVerifyDependencyError` if all of the given verifiers 283 report failure. 284 285 @param verifiers A list of `_StubVerifier` objects that are 286 expected to fail. 287 288 @return A set of `_DependencyFailure` objects. 289 """ 290 failures = [repair._DependencyFailure(v.description, v.message, v.tag) 291 for v in verifiers] 292 return set(failures) 293 294 295 def _generate_silent(self): 296 """ 297 Iterator to test different settings of the `silent` parameter. 298 299 This iterator exists to standardize testing assertions that 300 This iterator exists to standardize testing common 301 assertions about the `silent` parameter: 302 * When the parameter is true, no calls are made to the 303 `record()` method on the target host. 304 * When the parameter is false, certain expected calls are made 305 to the `record()` method on the target host. 306 307 The iterator is meant to be used like this: 308 309 for silent in self._generate_silent(): 310 # run test case that uses the silent parameter 311 self._check_log_records(silent, ... expected records ... ) 312 313 The code above will run its test case twice, once with 314 `silent=True` and once with `silent=False`. In between the 315 calls, log records are cleared. 316 317 @yields A boolean setting for `silent`. 318 """ 319 for silent in [False, True]: 320 yield silent 321 self._fake_host.reset_log_records() 322 323 324 def _check_log_records(self, silent, *record_data): 325 """ 326 Assert that log records occurred as expected. 327 328 Elements of `record_data` should be tuples of the form 329 `(tag, status)`, describing one expected log record. 330 The verifier or repair action for `tag` provides the expected 331 log record based on the status value. 332 333 The `silent` parameter is the value that was passed to the 334 verifier or repair action that did the logging. When true, 335 it indicates that no records should have been logged. 336 337 @param record_data List describing the expected record events. 338 @param silent When true, ignore `record_data` and assert 339 that nothing was logged. 340 """ 341 expected_records = [] 342 if not silent: 343 for tag, status in record_data: 344 expected_records.append( 345 self.nodes[tag].get_log_record(status)) 346 actual_records = self._fake_host.get_log_records() 347 self.assertEqual(expected_records, actual_records) 348 349 350class VerifyTests(_DependencyNodeTestCase): 351 """ 352 Unit tests for `Verifier`. 353 354 The tests in this class test the fundamental behaviors of the 355 `Verifier` class: 356 * Results from the `verify()` method are cached; the method is 357 only called the first time that `_verify_host()` is called. 358 * The `_verify_host()` method makes the expected calls to 359 `Host.record()` for every call to the `verify()` method. 360 * When a dependency fails, the dependent verifier isn't called. 361 * Verifier calls are made in the order required by the DAG. 362 363 The test cases don't use `RepairStrategy` to build DAG structures, 364 but instead rely on custom-built DAGs. 365 """ 366 367 def _generate_verify_count(self, verifier): 368 """ 369 Iterator to force a standard sequence with calls to `_reverify()`. 370 371 This iterator exists to standardize testing two common 372 assertions: 373 * The side effects from calling `_verify_host()` only 374 happen on the first call to the method, except... 375 * Calling `_reverify()` resets a verifier so that the 376 next call to `_verify_host()` will repeat the side 377 effects. 378 379 The iterator is meant to be used like this: 380 381 for count in self._generate_verify_cases(verifier): 382 # run a verifier._verify_host() test case 383 self.assertEqual(verifier.verify_count, count) 384 self._check_log_records(silent, ... expected records ... ) 385 386 The code above will run the `_verify_host()` test case twice, 387 then call `_reverify()` to clear cached results, then re-run 388 the test case two more times. 389 390 @param verifier The verifier to be tested and reverified. 391 @yields Each iteration yields the number of times `_reverify()` 392 has been called. 393 """ 394 for i in range(1, 3): 395 for _ in range(0, 2): 396 yield i 397 verifier._reverify() 398 self._fake_host.reset_log_records() 399 400 401 def test_success(self): 402 """ 403 Test proper handling of a successful verification. 404 405 Construct and call a simple, single-node verification that will 406 pass. Assert the following: 407 * The `verify()` method is called once. 408 * The expected 'GOOD' record is logged via `Host.record()`. 409 * If `_verify_host()` is called more than once, there are no 410 visible side-effects after the first call. 411 * Calling `_reverify()` clears all cached results. 412 """ 413 for silent in self._generate_silent(): 414 verifier = self._make_verifier(0, 'pass', []) 415 for count in self._generate_verify_count(verifier): 416 verifier._verify_host(self._fake_host, silent) 417 self.assertEqual(verifier.verify_count, count) 418 self._check_log_records(silent, ('pass', 'GOOD')) 419 420 421 def test_fail(self): 422 """ 423 Test proper handling of verification failure. 424 425 Construct and call a simple, single-node verification that will 426 fail. Assert the following: 427 * The failure is reported with the actual exception raised 428 by the verifier. 429 * The `verify()` method is called once. 430 * The expected 'FAIL' record is logged via `Host.record()`. 431 * If `_verify_host()` is called more than once, there are no 432 visible side-effects after the first call. 433 * Calling `_reverify()` clears all cached results. 434 """ 435 for silent in self._generate_silent(): 436 verifier = self._make_verifier(1, 'fail', []) 437 for count in self._generate_verify_count(verifier): 438 with self.assertRaises(hosts.AutoservVerifyError) as e: 439 verifier._verify_host(self._fake_host, silent) 440 self.assertEqual(verifier.verify_count, count) 441 self.assertEqual(verifier.message, str(e.exception)) 442 self._check_log_records(silent, ('fail', 'FAIL')) 443 444 445 def test_dependency_success(self): 446 """ 447 Test proper handling of dependencies that succeed. 448 449 Construct and call a two-node verification with one node 450 dependent on the other, where both nodes will pass. Assert the 451 following: 452 * The `verify()` method for both nodes is called once. 453 * The expected 'GOOD' record is logged via `Host.record()` 454 for both nodes. 455 * If `_verify_host()` is called more than once, there are no 456 visible side-effects after the first call. 457 * Calling `_reverify()` clears all cached results. 458 """ 459 for silent in self._generate_silent(): 460 child = self._make_verifier(0, 'pass', []) 461 parent = self._make_verifier(0, 'parent', [child]) 462 for count in self._generate_verify_count(parent): 463 parent._verify_host(self._fake_host, silent) 464 self.assertEqual(parent.verify_count, count) 465 self.assertEqual(child.verify_count, count) 466 self._check_log_records(silent, 467 ('pass', 'GOOD'), 468 ('parent', 'GOOD')) 469 470 471 def test_dependency_fail(self): 472 """ 473 Test proper handling of dependencies that fail. 474 475 Construct and call a two-node verification with one node 476 dependent on the other, where the dependency will fail. Assert 477 the following: 478 * The verification exception is `AutoservVerifyDependencyError`, 479 and the exception argument is the description of the failed 480 node. 481 * The `verify()` method for the failing node is called once, 482 and for the other node, not at all. 483 * The expected 'FAIL' record is logged via `Host.record()` 484 for the single failed node. 485 * If `_verify_host()` is called more than once, there are no 486 visible side-effects after the first call. 487 * Calling `_reverify()` clears all cached results. 488 """ 489 for silent in self._generate_silent(): 490 child = self._make_verifier(1, 'fail', []) 491 parent = self._make_verifier(0, 'parent', [child]) 492 failures = self._make_expected_failures(child) 493 for count in self._generate_verify_count(parent): 494 expected_exception = hosts.AutoservVerifyDependencyError 495 with self.assertRaises(expected_exception) as e: 496 parent._verify_host(self._fake_host, silent) 497 self.assertEqual(e.exception.failures, failures) 498 self.assertEqual(child.verify_count, count) 499 self.assertEqual(parent.verify_count, 0) 500 self._check_log_records(silent, ('fail', 'FAIL')) 501 502 503 def test_two_dependencies_pass(self): 504 """ 505 Test proper handling with two passing dependencies. 506 507 Construct and call a three-node verification with one node 508 dependent on the other two, where all nodes will pass. Assert 509 the following: 510 * The `verify()` method for all nodes is called once. 511 * The expected 'GOOD' records are logged via `Host.record()` 512 for all three nodes. 513 * If `_verify_host()` is called more than once, there are no 514 visible side-effects after the first call. 515 * Calling `_reverify()` clears all cached results. 516 """ 517 for silent in self._generate_silent(): 518 left = self._make_verifier(0, 'left', []) 519 right = self._make_verifier(0, 'right', []) 520 top = self._make_verifier(0, 'top', [left, right]) 521 for count in self._generate_verify_count(top): 522 top._verify_host(self._fake_host, silent) 523 self.assertEqual(top.verify_count, count) 524 self.assertEqual(left.verify_count, count) 525 self.assertEqual(right.verify_count, count) 526 self._check_log_records(silent, 527 ('left', 'GOOD'), 528 ('right', 'GOOD'), 529 ('top', 'GOOD')) 530 531 532 def test_two_dependencies_fail(self): 533 """ 534 Test proper handling with two failing dependencies. 535 536 Construct and call a three-node verification with one node 537 dependent on the other two, where both dependencies will fail. 538 Assert the following: 539 * The verification exception is `AutoservVerifyDependencyError`, 540 and the exception argument has the descriptions of both the 541 failed nodes. 542 * The `verify()` method for each failing node is called once, 543 and for the parent node not at all. 544 * The expected 'FAIL' records are logged via `Host.record()` 545 for the failing nodes. 546 * If `_verify_host()` is called more than once, there are no 547 visible side-effects after the first call. 548 * Calling `_reverify()` clears all cached results. 549 """ 550 for silent in self._generate_silent(): 551 left = self._make_verifier(1, 'left', []) 552 right = self._make_verifier(1, 'right', []) 553 top = self._make_verifier(0, 'top', [left, right]) 554 failures = self._make_expected_failures(left, right) 555 for count in self._generate_verify_count(top): 556 expected_exception = hosts.AutoservVerifyDependencyError 557 with self.assertRaises(expected_exception) as e: 558 top._verify_host(self._fake_host, silent) 559 self.assertEqual(e.exception.failures, failures) 560 self.assertEqual(top.verify_count, 0) 561 self.assertEqual(left.verify_count, count) 562 self.assertEqual(right.verify_count, count) 563 self._check_log_records(silent, 564 ('left', 'FAIL'), 565 ('right', 'FAIL')) 566 567 568 def test_two_dependencies_mixed(self): 569 """ 570 Test proper handling with mixed dependencies. 571 572 Construct and call a three-node verification with one node 573 dependent on the other two, where one dependency will pass, 574 and one will fail. Assert the following: 575 * The verification exception is `AutoservVerifyDependencyError`, 576 and the exception argument has the descriptions of the 577 single failed node. 578 * The `verify()` method for each dependency is called once, 579 and for the parent node not at all. 580 * The expected 'GOOD' and 'FAIL' records are logged via 581 `Host.record()` for the dependencies. 582 * If `_verify_host()` is called more than once, there are no 583 visible side-effects after the first call. 584 * Calling `_reverify()` clears all cached results. 585 """ 586 for silent in self._generate_silent(): 587 left = self._make_verifier(1, 'left', []) 588 right = self._make_verifier(0, 'right', []) 589 top = self._make_verifier(0, 'top', [left, right]) 590 failures = self._make_expected_failures(left) 591 for count in self._generate_verify_count(top): 592 expected_exception = hosts.AutoservVerifyDependencyError 593 with self.assertRaises(expected_exception) as e: 594 top._verify_host(self._fake_host, silent) 595 self.assertEqual(e.exception.failures, failures) 596 self.assertEqual(top.verify_count, 0) 597 self.assertEqual(left.verify_count, count) 598 self.assertEqual(right.verify_count, count) 599 self._check_log_records(silent, 600 ('left', 'FAIL'), 601 ('right', 'GOOD')) 602 603 604 def test_diamond_pass(self): 605 """ 606 Test a "diamond" structure DAG with all nodes passing. 607 608 Construct and call a "diamond" structure DAG where all nodes 609 will pass: 610 611 TOP 612 / \ 613 LEFT RIGHT 614 \ / 615 BOTTOM 616 617 Assert the following: 618 * The `verify()` method for all nodes is called once. 619 * The expected 'GOOD' records are logged via `Host.record()` 620 for all nodes. 621 * If `_verify_host()` is called more than once, there are no 622 visible side-effects after the first call. 623 * Calling `_reverify()` clears all cached results. 624 """ 625 for silent in self._generate_silent(): 626 bottom = self._make_verifier(0, 'bottom', []) 627 left = self._make_verifier(0, 'left', [bottom]) 628 right = self._make_verifier(0, 'right', [bottom]) 629 top = self._make_verifier(0, 'top', [left, right]) 630 for count in self._generate_verify_count(top): 631 top._verify_host(self._fake_host, silent) 632 self.assertEqual(top.verify_count, count) 633 self.assertEqual(left.verify_count, count) 634 self.assertEqual(right.verify_count, count) 635 self.assertEqual(bottom.verify_count, count) 636 self._check_log_records(silent, 637 ('bottom', 'GOOD'), 638 ('left', 'GOOD'), 639 ('right', 'GOOD'), 640 ('top', 'GOOD')) 641 642 643 def test_diamond_fail(self): 644 """ 645 Test a "diamond" structure DAG with the bottom node failing. 646 647 Construct and call a "diamond" structure DAG where the bottom 648 node will fail: 649 650 TOP 651 / \ 652 LEFT RIGHT 653 \ / 654 BOTTOM 655 656 Assert the following: 657 * The verification exception is `AutoservVerifyDependencyError`, 658 and the exception argument has the description of the 659 "bottom" node. 660 * The `verify()` method for the "bottom" node is called once, 661 and for the other nodes not at all. 662 * The expected 'FAIL' record is logged via `Host.record()` 663 for the "bottom" node. 664 * If `_verify_host()` is called more than once, there are no 665 visible side-effects after the first call. 666 * Calling `_reverify()` clears all cached results. 667 """ 668 for silent in self._generate_silent(): 669 bottom = self._make_verifier(1, 'bottom', []) 670 left = self._make_verifier(0, 'left', [bottom]) 671 right = self._make_verifier(0, 'right', [bottom]) 672 top = self._make_verifier(0, 'top', [left, right]) 673 failures = self._make_expected_failures(bottom) 674 for count in self._generate_verify_count(top): 675 expected_exception = hosts.AutoservVerifyDependencyError 676 with self.assertRaises(expected_exception) as e: 677 top._verify_host(self._fake_host, silent) 678 self.assertEqual(e.exception.failures, failures) 679 self.assertEqual(top.verify_count, 0) 680 self.assertEqual(left.verify_count, 0) 681 self.assertEqual(right.verify_count, 0) 682 self.assertEqual(bottom.verify_count, count) 683 self._check_log_records(silent, ('bottom', 'FAIL')) 684 685 686class RepairActionTests(_DependencyNodeTestCase): 687 """ 688 Unit tests for `RepairAction`. 689 690 The tests in this class test the fundamental behaviors of the 691 `RepairAction` class: 692 * Repair doesn't run unless all dependencies pass. 693 * Repair doesn't run unless at least one trigger fails. 694 * Repair reports the expected value of `status` for metrics. 695 * The `_repair_host()` method makes the expected calls to 696 `Host.record()` for every call to the `repair()` method. 697 698 The test cases don't use `RepairStrategy` to build repair 699 graphs, but instead rely on custom-built structures. 700 """ 701 702 def test_repair_not_triggered(self): 703 """ 704 Test a repair that doesn't trigger. 705 706 Construct and call a repair action with a verification trigger 707 that passes. Assert the following: 708 * The `verify()` method for the trigger is called. 709 * The `repair()` method is not called. 710 * The repair action's `status` field is 'untriggered'. 711 * The verifier logs the expected 'GOOD' message with 712 `Host.record()`. 713 * The repair action logs no messages with `Host.record()`. 714 """ 715 for silent in self._generate_silent(): 716 verifier = self._make_verifier(0, 'check', []) 717 repair_action = self._make_repair_action(True, 'unneeded', 718 [], [verifier]) 719 repair_action._repair_host(self._fake_host, silent) 720 self.assertEqual(verifier.verify_count, 1) 721 self.assertEqual(repair_action.repair_count, 0) 722 self.assertEqual(repair_action.status, 'skipped') 723 self._check_log_records(silent, ('check', 'GOOD')) 724 725 726 def test_repair_fails(self): 727 """ 728 Test a repair that triggers and fails. 729 730 Construct and call a repair action with a verification trigger 731 that fails. The repair fails by raising `_StubRepairFailure`. 732 Assert the following: 733 * The repair action fails with the `_StubRepairFailure` raised 734 by `repair()`. 735 * The `verify()` method for the trigger is called once. 736 * The `repair()` method is called once. 737 * The repair action's `status` field is 'failed-action'. 738 * The expected 'START', 'FAIL', and 'END FAIL' messages are 739 logged with `Host.record()` for the failed verifier and the 740 failed repair. 741 """ 742 for silent in self._generate_silent(): 743 verifier = self._make_verifier(1, 'fail', []) 744 repair_action = self._make_repair_action(False, 'nofix', 745 [], [verifier]) 746 with self.assertRaises(_StubRepairFailure) as e: 747 repair_action._repair_host(self._fake_host, silent) 748 self.assertEqual(repair_action.message, str(e.exception)) 749 self.assertEqual(verifier.verify_count, 1) 750 self.assertEqual(repair_action.repair_count, 1) 751 self.assertEqual(repair_action.status, 'repair_failure') 752 self._check_log_records(silent, 753 ('fail', 'FAIL'), 754 ('nofix', 'START'), 755 ('nofix', 'FAIL'), 756 ('nofix', 'END FAIL')) 757 758 759 def test_repair_success(self): 760 """ 761 Test a repair that fixes its trigger. 762 763 Construct and call a repair action that raises no exceptions, 764 using a repair trigger that fails first, then passes after 765 repair. Assert the following: 766 * The `repair()` method is called once. 767 * The trigger's `verify()` method is called twice. 768 * The repair action's `status` field is 'repaired'. 769 * The expected 'START', 'FAIL', 'GOOD', and 'END GOOD' 770 messages are logged with `Host.record()` for the verifier 771 and the repair. 772 """ 773 for silent in self._generate_silent(): 774 verifier = self._make_verifier(1, 'fail', []) 775 repair_action = self._make_repair_action(True, 'fix', 776 [], [verifier]) 777 repair_action._repair_host(self._fake_host, silent) 778 self.assertEqual(repair_action.repair_count, 1) 779 self.assertEqual(verifier.verify_count, 2) 780 self.assertEqual(repair_action.status, 'repaired') 781 self._check_log_records(silent, 782 ('fail', 'FAIL'), 783 ('fix', 'START'), 784 ('fail', 'GOOD'), 785 ('fix', 'END GOOD')) 786 787 788 def test_repair_noop(self): 789 """ 790 Test a repair that doesn't fix a failing trigger. 791 792 Construct and call a repair action with a trigger that fails. 793 The repair action raises no exceptions, and after repair, the 794 trigger still fails. Assert the following: 795 * The `_repair_host()` call fails with `AutoservRepairError`. 796 * The `repair()` method is called once. 797 * The trigger's `verify()` method is called twice. 798 * The repair action's `status` field is 'failed-trigger'. 799 * The expected 'START', 'FAIL', and 'END FAIL' messages are 800 logged with `Host.record()` for the verifier and the repair. 801 """ 802 for silent in self._generate_silent(): 803 verifier = self._make_verifier(2, 'fail', []) 804 repair_action = self._make_repair_action(True, 'nofix', 805 [], [verifier]) 806 with self.assertRaises(hosts.AutoservRepairError) as e: 807 repair_action._repair_host(self._fake_host, silent) 808 self.assertEqual(repair_action.repair_count, 1) 809 self.assertEqual(verifier.verify_count, 2) 810 self.assertEqual(repair_action.status, 'verify_failure') 811 self._check_log_records(silent, 812 ('fail', 'FAIL'), 813 ('nofix', 'START'), 814 ('fail', 'FAIL'), 815 ('nofix', 'END FAIL')) 816 817 818 def test_dependency_pass(self): 819 """ 820 Test proper handling of repair dependencies that pass. 821 822 Construct and call a repair action with a dependency and a 823 trigger. The dependency will pass and the trigger will fail and 824 be repaired. Assert the following: 825 * Repair passes. 826 * The `verify()` method for the dependency is called once. 827 * The `verify()` method for the trigger is called twice. 828 * The `repair()` method is called once. 829 * The repair action's `status` field is 'repaired'. 830 * The expected records are logged via `Host.record()` 831 for the successful dependency, the failed trigger, and 832 the successful repair. 833 """ 834 for silent in self._generate_silent(): 835 dep = self._make_verifier(0, 'dep', []) 836 trigger = self._make_verifier(1, 'trig', []) 837 repair = self._make_repair_action(True, 'fixit', 838 [dep], [trigger]) 839 repair._repair_host(self._fake_host, silent) 840 self.assertEqual(dep.verify_count, 1) 841 self.assertEqual(trigger.verify_count, 2) 842 self.assertEqual(repair.repair_count, 1) 843 self.assertEqual(repair.status, 'repaired') 844 self._check_log_records(silent, 845 ('dep', 'GOOD'), 846 ('trig', 'FAIL'), 847 ('fixit', 'START'), 848 ('trig', 'GOOD'), 849 ('fixit', 'END GOOD')) 850 851 852 def test_dependency_fail(self): 853 """ 854 Test proper handling of repair dependencies that fail. 855 856 Construct and call a repair action with a dependency and a 857 trigger, both of which fail. Assert the following: 858 * Repair fails with `AutoservVerifyDependencyError`, 859 and the exception argument is the description of the failed 860 dependency. 861 * The `verify()` method for the failing dependency is called 862 once. 863 * The trigger and the repair action aren't invoked at all. 864 * The repair action's `status` field is 'blocked'. 865 * The expected 'FAIL' record is logged via `Host.record()` 866 for the single failed dependency. 867 """ 868 for silent in self._generate_silent(): 869 dep = self._make_verifier(1, 'dep', []) 870 trigger = self._make_verifier(1, 'trig', []) 871 repair = self._make_repair_action(True, 'fixit', 872 [dep], [trigger]) 873 expected_exception = hosts.AutoservVerifyDependencyError 874 with self.assertRaises(expected_exception) as e: 875 repair._repair_host(self._fake_host, silent) 876 self.assertEqual(e.exception.failures, 877 self._make_expected_failures(dep)) 878 self.assertEqual(dep.verify_count, 1) 879 self.assertEqual(trigger.verify_count, 0) 880 self.assertEqual(repair.repair_count, 0) 881 self.assertEqual(repair.status, 'blocked') 882 self._check_log_records(silent, ('dep', 'FAIL')) 883 884 885class _RepairStrategyTestCase(_DependencyNodeTestCase): 886 """Shared base class for testing `RepairStrategy` methods.""" 887 888 def _make_verify_data(self, *input_data): 889 """ 890 Create `verify_data` for the `RepairStrategy` constructor. 891 892 `RepairStrategy` expects `verify_data` as a list of tuples 893 of the form `(constructor, tag, deps)`. Each item in 894 `input_data` is a tuple of the form `(tag, count, deps)` that 895 creates one entry in the returned list of `verify_data` tuples 896 as follows: 897 * `count` is used to create a constructor function that calls 898 `self._make_verifier()` with that value plus plus the 899 arguments provided by the `RepairStrategy` constructor. 900 * `tag` and `deps` will be passed as-is to the `RepairStrategy` 901 constructor. 902 903 @param input_data A list of tuples, each representing one 904 tuple in the `verify_data` list. 905 @return A list suitable to be the `verify_data` parameter for 906 the `RepairStrategy` constructor. 907 """ 908 strategy_data = [] 909 for tag, count, deps in input_data: 910 construct = functools.partial(self._make_verifier, count) 911 strategy_data.append((construct, tag, deps)) 912 return strategy_data 913 914 915 def _make_repair_data(self, *input_data): 916 """ 917 Create `repair_data` for the `RepairStrategy` constructor. 918 919 `RepairStrategy` expects `repair_data` as a list of tuples 920 of the form `(constructor, tag, deps, triggers)`. Each item in 921 `input_data` is a tuple of the form `(tag, success, deps, triggers)` 922 that creates one entry in the returned list of `repair_data` 923 tuples as follows: 924 * `success` is used to create a constructor function that calls 925 `self._make_verifier()` with that value plus plus the 926 arguments provided by the `RepairStrategy` constructor. 927 * `tag`, `deps`, and `triggers` will be passed as-is to the 928 `RepairStrategy` constructor. 929 930 @param input_data A list of tuples, each representing one 931 tuple in the `repair_data` list. 932 @return A list suitable to be the `repair_data` parameter for 933 the `RepairStrategy` constructor. 934 """ 935 strategy_data = [] 936 for tag, success, deps, triggers in input_data: 937 construct = functools.partial(self._make_repair_action, success) 938 strategy_data.append((construct, tag, deps, triggers)) 939 return strategy_data 940 941 942 def _make_strategy(self, verify_input, repair_input): 943 """ 944 Create a `RepairStrategy` from the given arguments. 945 946 @param verify_input As for `input_data` in 947 `_make_verify_data()`. 948 @param repair_input As for `input_data` in 949 `_make_repair_data()`. 950 """ 951 verify_data = self._make_verify_data(*verify_input) 952 repair_data = self._make_repair_data(*repair_input) 953 return hosts.RepairStrategy(verify_data, repair_data, 'unittest') 954 955 def _check_silent_records(self, silent): 956 """ 957 Check that logging honored the `silent` parameter. 958 959 Asserts that logging with `Host.record()` occurred (or did not 960 occur) in accordance with the value of `silent`. 961 962 This method only asserts the presence or absence of log records. 963 Coverage for the contents of the log records is handled in other 964 test cases. 965 966 @param silent When true, there should be no log records; 967 otherwise there should be records present. 968 """ 969 log_records = self._fake_host.get_log_records() 970 if silent: 971 self.assertEqual(log_records, []) 972 else: 973 self.assertNotEqual(log_records, []) 974 975 976class RepairStrategyVerifyTests(_RepairStrategyTestCase): 977 """ 978 Unit tests for `RepairStrategy.verify()`. 979 980 These unit tests focus on verifying that the `RepairStrategy` 981 constructor creates the expected DAG structure from given 982 `verify_data`. Functional testing here is mainly confined to 983 asserting that `RepairStrategy.verify()` properly distinguishes 984 success from failure. Testing the behavior of specific DAG 985 structures is left to tests in `VerifyTests`. 986 """ 987 988 def test_single_node(self): 989 """ 990 Test construction of a single-node verification DAG. 991 992 Assert that the structure looks like this: 993 994 Root Node -> Main Node 995 """ 996 verify_data = self._make_verify_data(('main', 0, ())) 997 strategy = hosts.RepairStrategy(verify_data, [], 'unittest') 998 verifier = self.nodes['main'] 999 self.assertEqual( 1000 strategy._verify_root._dependency_list, 1001 [verifier]) 1002 self.assertEqual(verifier._dependency_list, []) 1003 1004 1005 def test_single_dependency(self): 1006 """ 1007 Test construction of a two-node dependency chain. 1008 1009 Assert that the structure looks like this: 1010 1011 Root Node -> Parent Node -> Child Node 1012 """ 1013 verify_data = self._make_verify_data( 1014 ('child', 0, ()), 1015 ('parent', 0, ('child',))) 1016 strategy = hosts.RepairStrategy(verify_data, [], 'unittest') 1017 parent = self.nodes['parent'] 1018 child = self.nodes['child'] 1019 self.assertEqual( 1020 strategy._verify_root._dependency_list, [parent]) 1021 self.assertEqual( 1022 parent._dependency_list, [child]) 1023 self.assertEqual( 1024 child._dependency_list, []) 1025 1026 1027 def test_two_nodes_and_dependency(self): 1028 """ 1029 Test construction of two nodes with a shared dependency. 1030 1031 Assert that the structure looks like this: 1032 1033 Root Node -> Left Node ---\ 1034 \ -> Bottom Node 1035 -> Right Node / 1036 """ 1037 verify_data = self._make_verify_data( 1038 ('bottom', 0, ()), 1039 ('left', 0, ('bottom',)), 1040 ('right', 0, ('bottom',))) 1041 strategy = hosts.RepairStrategy(verify_data, [], 'unittest') 1042 bottom = self.nodes['bottom'] 1043 left = self.nodes['left'] 1044 right = self.nodes['right'] 1045 self.assertEqual( 1046 strategy._verify_root._dependency_list, 1047 [left, right]) 1048 self.assertEqual(left._dependency_list, [bottom]) 1049 self.assertEqual(right._dependency_list, [bottom]) 1050 self.assertEqual(bottom._dependency_list, []) 1051 1052 1053 def test_three_nodes(self): 1054 """ 1055 Test construction of three nodes with no dependencies. 1056 1057 Assert that the structure looks like this: 1058 1059 -> Node One 1060 / 1061 Root Node -> Node Two 1062 \ 1063 -> Node Three 1064 1065 N.B. This test exists to enforce ordering expectations of 1066 root-level DAG nodes. Three nodes are used to make it unlikely 1067 that randomly ordered roots will match expectations. 1068 """ 1069 verify_data = self._make_verify_data( 1070 ('one', 0, ()), 1071 ('two', 0, ()), 1072 ('three', 0, ())) 1073 strategy = hosts.RepairStrategy(verify_data, [], 'unittest') 1074 one = self.nodes['one'] 1075 two = self.nodes['two'] 1076 three = self.nodes['three'] 1077 self.assertEqual( 1078 strategy._verify_root._dependency_list, 1079 [one, two, three]) 1080 self.assertEqual(one._dependency_list, []) 1081 self.assertEqual(two._dependency_list, []) 1082 self.assertEqual(three._dependency_list, []) 1083 1084 1085 def test_verify(self): 1086 """ 1087 Test behavior of the `verify()` method. 1088 1089 Build a `RepairStrategy` with a single verifier. Assert the 1090 following: 1091 * If the verifier passes, `verify()` passes. 1092 * If the verifier fails, `verify()` fails. 1093 * The verifier is reinvoked with every call to `verify()`; 1094 cached results are not re-used. 1095 """ 1096 verify_data = self._make_verify_data(('tester', 0, ())) 1097 strategy = hosts.RepairStrategy(verify_data, [], 'unittest') 1098 verifier = self.nodes['tester'] 1099 count = 0 1100 for silent in self._generate_silent(): 1101 for i in range(0, 2): 1102 for j in range(0, 2): 1103 strategy.verify(self._fake_host, silent) 1104 self._check_silent_records(silent) 1105 count += 1 1106 self.assertEqual(verifier.verify_count, count) 1107 verifier.unrepair() 1108 for j in range(0, 2): 1109 with self.assertRaises(Exception) as e: 1110 strategy.verify(self._fake_host, silent) 1111 self._check_silent_records(silent) 1112 count += 1 1113 self.assertEqual(verifier.verify_count, count) 1114 verifier.try_repair() 1115 1116 1117class RepairStrategyRepairTests(_RepairStrategyTestCase): 1118 """ 1119 Unit tests for `RepairStrategy.repair()`. 1120 1121 These unit tests focus on verifying that the `RepairStrategy` 1122 constructor creates the expected repair list from given 1123 `repair_data`. Functional testing here is confined to asserting 1124 that `RepairStrategy.repair()` properly distinguishes success from 1125 failure. Testing the behavior of specific repair structures is left 1126 to tests in `RepairActionTests`. 1127 """ 1128 1129 def _check_common_trigger(self, strategy, repair_tags, triggers): 1130 self.assertEqual(strategy._repair_actions, 1131 [self.nodes[tag] for tag in repair_tags]) 1132 for tag in repair_tags: 1133 self.assertEqual(self.nodes[tag]._trigger_list, 1134 triggers) 1135 self.assertEqual(self.nodes[tag]._dependency_list, []) 1136 1137 1138 def test_single_repair_with_trigger(self): 1139 """ 1140 Test constructing a strategy with a single repair trigger. 1141 1142 Build a `RepairStrategy` with a single repair action and a 1143 single trigger. Assert that the trigger graph looks like this: 1144 1145 Repair -> Trigger 1146 1147 Assert that there are no repair dependencies. 1148 """ 1149 verify_input = (('base', 0, ()),) 1150 repair_input = (('fixit', True, (), ('base',)),) 1151 strategy = self._make_strategy(verify_input, repair_input) 1152 self._check_common_trigger(strategy, 1153 ['fixit'], 1154 [self.nodes['base']]) 1155 1156 1157 def test_repair_with_root_trigger(self): 1158 """ 1159 Test construction of a repair triggering on the root verifier. 1160 1161 Build a `RepairStrategy` with a single repair action that 1162 triggers on the root verifier. Assert that the trigger graph 1163 looks like this: 1164 1165 Repair -> Root Verifier 1166 1167 Assert that there are no repair dependencies. 1168 """ 1169 root_tag = hosts.RepairStrategy.ROOT_TAG 1170 repair_input = (('fixit', True, (), (root_tag,)),) 1171 strategy = self._make_strategy([], repair_input) 1172 self._check_common_trigger(strategy, 1173 ['fixit'], 1174 [strategy._verify_root]) 1175 1176 1177 def test_three_repairs(self): 1178 """ 1179 Test constructing a strategy with three repair actions. 1180 1181 Build a `RepairStrategy` with a three repair actions sharing a 1182 single trigger. Assert that the trigger graph looks like this: 1183 1184 Repair A -> Trigger 1185 Repair B -> Trigger 1186 Repair C -> Trigger 1187 1188 Assert that there are no repair dependencies. 1189 1190 N.B. This test exists to enforce ordering expectations of 1191 repair nodes. Three nodes are used to make it unlikely that 1192 randomly ordered actions will match expectations. 1193 """ 1194 verify_input = (('base', 0, ()),) 1195 repair_tags = ['a', 'b', 'c'] 1196 repair_input = ( 1197 (tag, True, (), ('base',)) for tag in repair_tags) 1198 strategy = self._make_strategy(verify_input, repair_input) 1199 self._check_common_trigger(strategy, 1200 repair_tags, 1201 [self.nodes['base']]) 1202 1203 1204 def test_repair_dependency(self): 1205 """ 1206 Test construction of a repair with a dependency. 1207 1208 Build a `RepairStrategy` with a single repair action that 1209 depends on a single verifier. Assert that the dependency graph 1210 looks like this: 1211 1212 Repair -> Verifier 1213 1214 Assert that there are no repair triggers. 1215 """ 1216 verify_input = (('base', 0, ()),) 1217 repair_input = (('fixit', True, ('base',), ()),) 1218 strategy = self._make_strategy(verify_input, repair_input) 1219 self.assertEqual(strategy._repair_actions, 1220 [self.nodes['fixit']]) 1221 self.assertEqual(self.nodes['fixit']._trigger_list, []) 1222 self.assertEqual(self.nodes['fixit']._dependency_list, 1223 [self.nodes['base']]) 1224 1225 1226 def _check_repair_failure(self, strategy, silent): 1227 """ 1228 Check the effects of a call to `repair()` that fails. 1229 1230 For the given strategy object, call the `repair()` method; the 1231 call is expected to fail and all repair actions are expected to 1232 trigger. 1233 1234 Assert the following: 1235 * The call raises an exception. 1236 * For each repair action in the strategy, its `repair()` 1237 method is called exactly once. 1238 1239 @param strategy The strategy to be tested. 1240 """ 1241 action_counts = [(a, a.repair_count) 1242 for a in strategy._repair_actions] 1243 with self.assertRaises(Exception) as e: 1244 strategy.repair(self._fake_host, silent) 1245 self._check_silent_records(silent) 1246 for action, count in action_counts: 1247 self.assertEqual(action.repair_count, count + 1) 1248 1249 1250 def _check_repair_success(self, strategy, silent): 1251 """ 1252 Check the effects of a call to `repair()` that succeeds. 1253 1254 For the given strategy object, call the `repair()` method; the 1255 call is expected to succeed without raising an exception and all 1256 repair actions are expected to trigger. 1257 1258 Assert that for each repair action in the strategy, its 1259 `repair()` method is called exactly once. 1260 1261 @param strategy The strategy to be tested. 1262 """ 1263 action_counts = [(a, a.repair_count) 1264 for a in strategy._repair_actions] 1265 strategy.repair(self._fake_host, silent) 1266 self._check_silent_records(silent) 1267 for action, count in action_counts: 1268 self.assertEqual(action.repair_count, count + 1) 1269 1270 1271 def test_repair(self): 1272 """ 1273 Test behavior of the `repair()` method. 1274 1275 Build a `RepairStrategy` with two repair actions each depending 1276 on its own verifier. Set up calls to `repair()` for each of 1277 the following conditions: 1278 * Both repair actions trigger and fail. 1279 * Both repair actions trigger and succeed. 1280 * Both repair actions trigger; the first one fails, but the 1281 second one succeeds. 1282 * Both repair actions trigger; the first one succeeds, but the 1283 second one fails. 1284 1285 Assert the following: 1286 * When both repair actions succeed, `repair()` succeeds. 1287 * When either repair action fails, `repair()` fails. 1288 * After each call to the strategy's `repair()` method, each 1289 repair action triggered exactly once. 1290 """ 1291 verify_input = (('a', 2, ()), ('b', 2, ())) 1292 repair_input = (('afix', True, (), ('a',)), 1293 ('bfix', True, (), ('b',))) 1294 strategy = self._make_strategy(verify_input, repair_input) 1295 1296 for silent in self._generate_silent(): 1297 # call where both 'afix' and 'bfix' fail 1298 self._check_repair_failure(strategy, silent) 1299 # repair counts are now 1 for both verifiers 1300 1301 # call where both 'afix' and 'bfix' succeed 1302 self._check_repair_success(strategy, silent) 1303 # repair counts are now 0 for both verifiers 1304 1305 # call where 'afix' fails and 'bfix' succeeds 1306 for tag in ['a', 'a', 'b']: 1307 self.nodes[tag].unrepair() 1308 self._check_repair_failure(strategy, silent) 1309 # 'a' repair count is 1; 'b' count is 0 1310 1311 # call where 'afix' succeeds and 'bfix' fails 1312 for tag in ['b', 'b']: 1313 self.nodes[tag].unrepair() 1314 self._check_repair_failure(strategy, silent) 1315 # 'a' repair count is 0; 'b' count is 1 1316 1317 for tag in ['a', 'a', 'b']: 1318 self.nodes[tag].unrepair() 1319 # repair counts are now 2 for both verifiers 1320 1321 1322if __name__ == '__main__': 1323 unittest.main() 1324