1# Copyright 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import Queue 6import datetime 7import logging 8import os 9import shutil 10import signal 11import sys 12import tempfile 13import time 14import unittest 15 16import mox 17 18import common 19import gs_offloader 20import job_directories 21 22from autotest_lib.client.common_lib import global_config 23from autotest_lib.client.common_lib import time_utils 24from autotest_lib.client.common_lib import utils 25from autotest_lib.scheduler import email_manager 26 27 28# Test value to use for `days_old`, if nothing else is required. 29_TEST_EXPIRATION_AGE = 7 30 31# When constructing sample time values for testing expiration, 32# allow this many seconds between the expiration time and the 33# current time. 34_MARGIN_SECS = 10.0 35 36 37def _get_options(argv): 38 """Helper function to exercise command line parsing. 39 40 @param argv Value of sys.argv to be parsed. 41 42 """ 43 sys.argv = ['bogus.py'] + argv 44 return gs_offloader.parse_options() 45 46 47class OffloaderOptionsTests(mox.MoxTestBase): 48 """Tests for the `Offloader` constructor. 49 50 Tests that offloader instance fields are set as expected 51 for given command line options. 52 53 """ 54 55 _REGULAR_ONLY = set([job_directories.RegularJobDirectory]) 56 _SPECIAL_ONLY = set([job_directories.SpecialJobDirectory]) 57 _BOTH = _REGULAR_ONLY | _SPECIAL_ONLY 58 59 60 def setUp(self): 61 super(OffloaderOptionsTests, self).setUp() 62 self.mox.StubOutWithMock(utils, 'get_offload_gsuri') 63 gs_offloader.GS_OFFLOADING_ENABLED = True 64 65 66 def _mock_get_offload_func(self, is_moblab): 67 """Mock the process of getting the offload_dir function.""" 68 if is_moblab: 69 expected_gsuri = '%sresults/%s/%s/' % ( 70 global_config.global_config.get_config_value( 71 'CROS', 'image_storage_server'), 72 'Fa:ke:ma:c0:12:34', 'rand0m-uu1d') 73 else: 74 expected_gsuri = utils.DEFAULT_OFFLOAD_GSURI 75 utils.get_offload_gsuri().AndReturn(expected_gsuri) 76 offload_func = gs_offloader.get_offload_dir_func(expected_gsuri, False) 77 self.mox.StubOutWithMock(gs_offloader, 'get_offload_dir_func') 78 gs_offloader.get_offload_dir_func(expected_gsuri, False).AndReturn( 79 offload_func) 80 self.mox.ReplayAll() 81 return offload_func 82 83 84 def test_process_no_options(self): 85 """Test default offloader options.""" 86 offload_func = self._mock_get_offload_func(False) 87 offloader = gs_offloader.Offloader(_get_options([])) 88 self.assertEqual(set(offloader._jobdir_classes), 89 self._REGULAR_ONLY) 90 self.assertEqual(offloader._processes, 1) 91 self.assertEqual(offloader._offload_func, 92 offload_func) 93 self.assertEqual(offloader._age_limit, 0) 94 95 96 def test_process_all_option(self): 97 """Test offloader handling for the --all option.""" 98 offload_func = self._mock_get_offload_func(False) 99 offloader = gs_offloader.Offloader(_get_options(['--all'])) 100 self.assertEqual(set(offloader._jobdir_classes), self._BOTH) 101 self.assertEqual(offloader._processes, 1) 102 self.assertEqual(offloader._offload_func, 103 offload_func) 104 self.assertEqual(offloader._age_limit, 0) 105 106 107 def test_process_hosts_option(self): 108 """Test offloader handling for the --hosts option.""" 109 offload_func = self._mock_get_offload_func(False) 110 offloader = gs_offloader.Offloader( 111 _get_options(['--hosts'])) 112 self.assertEqual(set(offloader._jobdir_classes), 113 self._SPECIAL_ONLY) 114 self.assertEqual(offloader._processes, 1) 115 self.assertEqual(offloader._offload_func, 116 offload_func) 117 self.assertEqual(offloader._age_limit, 0) 118 119 120 def test_parallelism_option(self): 121 """Test offloader handling for the --parallelism option.""" 122 offload_func = self._mock_get_offload_func(False) 123 offloader = gs_offloader.Offloader( 124 _get_options(['--parallelism', '2'])) 125 self.assertEqual(set(offloader._jobdir_classes), 126 self._REGULAR_ONLY) 127 self.assertEqual(offloader._processes, 2) 128 self.assertEqual(offloader._offload_func, 129 offload_func) 130 self.assertEqual(offloader._age_limit, 0) 131 132 133 def test_delete_only_option(self): 134 """Test offloader handling for the --delete_only option.""" 135 offloader = gs_offloader.Offloader( 136 _get_options(['--delete_only'])) 137 self.assertEqual(set(offloader._jobdir_classes), 138 self._REGULAR_ONLY) 139 self.assertEqual(offloader._processes, 1) 140 self.assertEqual(offloader._offload_func, 141 gs_offloader.delete_files) 142 self.assertEqual(offloader._age_limit, 0) 143 144 145 def test_days_old_option(self): 146 """Test offloader handling for the --days_old option.""" 147 offload_func = self._mock_get_offload_func(False) 148 offloader = gs_offloader.Offloader( 149 _get_options(['--days_old', '7'])) 150 self.assertEqual(set(offloader._jobdir_classes), 151 self._REGULAR_ONLY) 152 self.assertEqual(offloader._processes, 1) 153 self.assertEqual(offloader._offload_func, 154 offload_func) 155 self.assertEqual(offloader._age_limit, 7) 156 157 158 def test_moblab_gsuri_generation(self): 159 """Test offloader construction for Moblab.""" 160 offload_func = self._mock_get_offload_func(True) 161 offloader = gs_offloader.Offloader(_get_options([])) 162 self.assertEqual(set(offloader._jobdir_classes), 163 self._REGULAR_ONLY) 164 self.assertEqual(offloader._processes, 1) 165 self.assertEqual(offloader._offload_func, 166 offload_func) 167 self.assertEqual(offloader._age_limit, 0) 168 169 170 def test_globalconfig_offloading_flag(self): 171 """Test enabling of --delete_only via global_config.""" 172 gs_offloader.GS_OFFLOADING_ENABLED = False 173 offloader = gs_offloader.Offloader( 174 _get_options([])) 175 self.assertEqual(offloader._offload_func, 176 gs_offloader.delete_files) 177 178 179def _make_timestamp(age_limit, is_expired): 180 """Create a timestamp for use by `job_directories._is_job_expired()`. 181 182 The timestamp will meet the syntactic requirements for 183 timestamps used as input to `_is_job_expired()`. If 184 `is_expired` is true, the timestamp will be older than 185 `age_limit` days before the current time; otherwise, the 186 date will be younger. 187 188 @param age_limit The number of days before expiration of the 189 target timestamp. 190 @param is_expired Whether the timestamp should be expired 191 relative to `age_limit`. 192 193 """ 194 seconds = -_MARGIN_SECS 195 if is_expired: 196 seconds = -seconds 197 delta = datetime.timedelta(days=age_limit, seconds=seconds) 198 reference_time = datetime.datetime.now() - delta 199 return reference_time.strftime(time_utils.TIME_FMT) 200 201 202class JobExpirationTests(unittest.TestCase): 203 """Tests to exercise `job_directories._is_job_expired()`.""" 204 205 def test_expired(self): 206 """Test detection of an expired job.""" 207 timestamp = _make_timestamp(_TEST_EXPIRATION_AGE, True) 208 self.assertTrue( 209 job_directories._is_job_expired( 210 _TEST_EXPIRATION_AGE, timestamp)) 211 212 213 def test_alive(self): 214 """Test detection of a job that's not expired.""" 215 # N.B. This test may fail if its run time exceeds more than 216 # about _MARGIN_SECS seconds. 217 timestamp = _make_timestamp(_TEST_EXPIRATION_AGE, False) 218 self.assertFalse( 219 job_directories._is_job_expired( 220 _TEST_EXPIRATION_AGE, timestamp)) 221 222 223class _MockJobDirectory(job_directories._JobDirectory): 224 """Subclass of `_JobDirectory` used as a helper for tests.""" 225 226 GLOB_PATTERN = '[0-9]*-*' 227 228 229 def __init__(self, resultsdir): 230 """Create new job in initial state.""" 231 super(_MockJobDirectory, self).__init__(resultsdir) 232 self._timestamp = None 233 self.queue_args = [resultsdir, os.path.dirname(resultsdir)] 234 235 236 def get_timestamp_if_finished(self): 237 return self._timestamp 238 239 240 def set_finished(self, days_old): 241 """Make this job appear to be finished. 242 243 After calling this function, calls to `enqueue_offload()` 244 will find this job as finished, but not expired and ready 245 for offload. Note that when `days_old` is 0, 246 `enqueue_offload()` will treat a finished job as eligible 247 for offload. 248 249 @param days_old The value of the `days_old` parameter that 250 will be passed to `enqueue_offload()` for 251 testing. 252 253 """ 254 self._timestamp = _make_timestamp(days_old, False) 255 256 257 def set_expired(self, days_old): 258 """Make this job eligible to be offloaded. 259 260 After calling this function, calls to `offload` will attempt 261 to offload this job. 262 263 @param days_old The value of the `days_old` parameter that 264 will be passed to `enqueue_offload()` for 265 testing. 266 267 """ 268 self._timestamp = _make_timestamp(days_old, True) 269 270 271 def set_incomplete(self): 272 """Make this job appear to have failed offload just once.""" 273 self._offload_count += 1 274 self._first_offload_start = time.time() 275 if not os.path.isdir(self._dirname): 276 os.mkdir(self._dirname) 277 278 279 def set_reportable(self): 280 """Make this job be reportable.""" 281 self.set_incomplete() 282 self._offload_count += 1 283 284 285 def set_complete(self): 286 """Make this job be completed.""" 287 self._offload_count += 1 288 if os.path.isdir(self._dirname): 289 os.rmdir(self._dirname) 290 291 292 def process_gs_instructions(self): 293 """Always still offload the job directory.""" 294 return True 295 296 297class CommandListTests(unittest.TestCase): 298 """Tests for `get_cmd_list()`.""" 299 300 def _command_list_assertions(self, job, use_rsync=True, multi=False): 301 """Call `get_cmd_list()` and check the return value. 302 303 Check the following assertions: 304 * The command name (argv[0]) is 'gsutil'. 305 * '-m' option (argv[1]) is on when the argument, multi, is True. 306 * The arguments contain the 'cp' subcommand. 307 * The next-to-last argument (the source directory) is the 308 job's `queue_args[0]`. 309 * The last argument (the destination URL) is the job's 310 'queue_args[1]'. 311 312 @param job A job with properly calculated arguments to 313 `get_cmd_list()` 314 @param use_rsync True when using 'rsync'. False when using 'cp'. 315 @param multi True when using '-m' option for gsutil. 316 317 """ 318 test_bucket_uri = 'gs://a-test-bucket' 319 320 gs_offloader.USE_RSYNC_ENABLED = use_rsync 321 322 command = gs_offloader.get_cmd_list( 323 multi, job.queue_args[0], 324 os.path.join(test_bucket_uri, job.queue_args[1])) 325 326 self.assertEqual(command[0], 'gsutil') 327 if multi: 328 self.assertEqual(command[1], '-m') 329 self.assertEqual(command[-2], job.queue_args[0]) 330 331 if use_rsync: 332 self.assertTrue('rsync' in command) 333 self.assertEqual(command[-1], 334 os.path.join(test_bucket_uri, job.queue_args[0])) 335 else: 336 self.assertTrue('cp' in command) 337 self.assertEqual(command[-1], 338 os.path.join(test_bucket_uri, job.queue_args[1])) 339 340 341 def test_get_cmd_list_regular(self): 342 """Test `get_cmd_list()` as for a regular job.""" 343 job = _MockJobDirectory('118-debug') 344 self._command_list_assertions(job) 345 346 347 def test_get_cmd_list_special(self): 348 """Test `get_cmd_list()` as for a special job.""" 349 job = _MockJobDirectory('hosts/host1/118-reset') 350 self._command_list_assertions(job) 351 352 353 def test_get_cmd_list_regular_no_rsync(self): 354 """Test `get_cmd_list()` as for a regular job.""" 355 job = _MockJobDirectory('118-debug') 356 self._command_list_assertions(job, use_rsync=False) 357 358 359 def test_get_cmd_list_special_no_rsync(self): 360 """Test `get_cmd_list()` as for a special job.""" 361 job = _MockJobDirectory('hosts/host1/118-reset') 362 self._command_list_assertions(job, use_rsync=False) 363 364 365 def test_get_cmd_list_regular_multi(self): 366 """Test `get_cmd_list()` as for a regular job with True multi.""" 367 job = _MockJobDirectory('118-debug') 368 self._command_list_assertions(job, multi=True) 369 370 371 def test_get_cmd_list_special_multi(self): 372 """Test `get_cmd_list()` as for a special job with True multi.""" 373 job = _MockJobDirectory('hosts/host1/118-reset') 374 self._command_list_assertions(job, multi=True) 375 376 377# Below is partial sample of e-mail notification text. This text is 378# deliberately hard-coded and then parsed to create the test data; 379# the idea is to make sure the actual text format will be reviewed 380# by a human being. 381# 382# first offload count directory 383# --+----1----+---- ----+ ----+----1----+----2----+----3 384_SAMPLE_DIRECTORIES_REPORT = '''\ 385=================== ====== ============================== 3862014-03-14 15:09:26 1 118-fubar 3872014-03-14 15:19:23 2 117-fubar 3882014-03-14 15:29:20 6 116-fubar 3892014-03-14 15:39:17 24 115-fubar 3902014-03-14 15:49:14 120 114-fubar 3912014-03-14 15:59:11 720 113-fubar 3922014-03-14 16:09:08 5040 112-fubar 3932014-03-14 16:19:05 40320 111-fubar 394''' 395 396 397class EmailTemplateTests(mox.MoxTestBase): 398 """Test the formatting of e-mail notifications.""" 399 400 def setUp(self): 401 super(EmailTemplateTests, self).setUp() 402 self.mox.StubOutWithMock(email_manager.manager, 403 'send_email') 404 self._joblist = [] 405 for line in _SAMPLE_DIRECTORIES_REPORT.split('\n')[1 : -1]: 406 date_, time_, count, dir_ = line.split() 407 job = _MockJobDirectory(dir_) 408 job._offload_count = int(count) 409 timestruct = time.strptime("%s %s" % (date_, time_), 410 gs_offloader.ERROR_EMAIL_TIME_FORMAT) 411 job._first_offload_start = time.mktime(timestruct) 412 # enter the jobs in reverse order, to make sure we 413 # test that the output will be sorted. 414 self._joblist.insert(0, job) 415 416 417 def test_email_template(self): 418 """Trigger an e-mail report and check its contents.""" 419 # The last line of the report is a separator that we 420 # repeat in the first line of our expected result data. 421 # So, we remove that separator from the end of the of 422 # the e-mail report message. 423 # 424 # The last element in the list returned by split('\n') 425 # will be an empty string, so to remove the separator, 426 # we remove the next-to-last entry in the list. 427 report_lines = gs_offloader.ERROR_EMAIL_REPORT_FORMAT.split('\n') 428 expected_message = ('\n'.join(report_lines[: -2] + 429 report_lines[-1 :]) + 430 _SAMPLE_DIRECTORIES_REPORT) 431 email_manager.manager.send_email( 432 mox.IgnoreArg(), mox.IgnoreArg(), expected_message) 433 self.mox.ReplayAll() 434 gs_offloader.report_offload_failures(self._joblist) 435 436 437 def test_email_url(self): 438 """Check that the expected helper url is in the email header.""" 439 self.assertIn(gs_offloader.ERROR_EMAIL_HELPER_URL, 440 gs_offloader.ERROR_EMAIL_REPORT_FORMAT) 441 442 443class _MockJob(object): 444 """Class to mock the return value of `AFE.get_jobs()`.""" 445 def __init__(self, created): 446 self.created_on = created 447 448 449class _MockHostQueueEntry(object): 450 """Class to mock the return value of `AFE.get_host_queue_entries()`.""" 451 def __init__(self, finished): 452 self.finished_on = finished 453 454 455class _MockSpecialTask(object): 456 """Class to mock the return value of `AFE.get_special_tasks()`.""" 457 def __init__(self, finished): 458 self.time_finished = finished 459 460 461class JobDirectorySubclassTests(mox.MoxTestBase): 462 """Test specific to RegularJobDirectory and SpecialJobDirectory. 463 464 This provides coverage for the implementation in both 465 RegularJobDirectory and SpecialJobDirectory. 466 467 """ 468 469 def setUp(self): 470 super(JobDirectorySubclassTests, self).setUp() 471 self.mox.StubOutWithMock(job_directories._AFE, 'get_jobs') 472 self.mox.StubOutWithMock(job_directories._AFE, 473 'get_host_queue_entries') 474 self.mox.StubOutWithMock(job_directories._AFE, 475 'get_special_tasks') 476 477 478 def test_regular_job_fields(self): 479 """Test the constructor for `RegularJobDirectory`. 480 481 Construct a regular job, and assert that the `_dirname` 482 and `_id` attributes are set as expected. 483 484 """ 485 resultsdir = '118-fubar' 486 job = job_directories.RegularJobDirectory(resultsdir) 487 self.assertEqual(job._dirname, resultsdir) 488 self.assertEqual(job._id, 118) 489 490 491 def test_special_job_fields(self): 492 """Test the constructor for `SpecialJobDirectory`. 493 494 Construct a special job, and assert that the `_dirname` 495 and `_id` attributes are set as expected. 496 497 """ 498 destdir = 'hosts/host1' 499 resultsdir = destdir + '/118-reset' 500 job = job_directories.SpecialJobDirectory(resultsdir) 501 self.assertEqual(job._dirname, resultsdir) 502 self.assertEqual(job._id, 118) 503 504 505 def _check_finished_job(self, jobtime, hqetimes, expected): 506 """Mock and test behavior of a finished job. 507 508 Initialize the mocks for a call to 509 `get_timestamp_if_finished()`, then simulate one call. 510 Assert that the returned timestamp matches the passed 511 in expected value. 512 513 @param jobtime Time used to construct a _MockJob object. 514 @param hqetimes List of times used to construct 515 _MockHostQueueEntry objects. 516 @param expected Expected time to be returned by 517 get_timestamp_if_finished 518 519 """ 520 job = job_directories.RegularJobDirectory('118-fubar') 521 job_directories._AFE.get_jobs( 522 id=job._id, finished=True).AndReturn( 523 [_MockJob(jobtime)]) 524 job_directories._AFE.get_host_queue_entries( 525 finished_on__isnull=False, 526 job_id=job._id).AndReturn( 527 [_MockHostQueueEntry(t) for t in hqetimes]) 528 self.mox.ReplayAll() 529 self.assertEqual(expected, job.get_timestamp_if_finished()) 530 self.mox.VerifyAll() 531 532 533 def test_finished_regular_job(self): 534 """Test getting the timestamp for a finished regular job. 535 536 Tests the return value for 537 `RegularJobDirectory.get_timestamp_if_finished()` when 538 the AFE indicates the job is finished. 539 540 """ 541 created_timestamp = _make_timestamp(1, True) 542 hqe_timestamp = _make_timestamp(0, True) 543 self._check_finished_job(created_timestamp, 544 [hqe_timestamp], 545 hqe_timestamp) 546 547 548 def test_finished_regular_job_multiple_hqes(self): 549 """Test getting the timestamp for a regular job with multiple hqes. 550 551 Tests the return value for 552 `RegularJobDirectory.get_timestamp_if_finished()` when 553 the AFE indicates the job is finished and the job has multiple host 554 queue entries. 555 556 Tests that the returned timestamp is the latest timestamp in 557 the list of HQEs, regardless of the returned order. 558 559 """ 560 created_timestamp = _make_timestamp(2, True) 561 older_hqe_timestamp = _make_timestamp(1, True) 562 newer_hqe_timestamp = _make_timestamp(0, True) 563 hqe_list = [older_hqe_timestamp, 564 newer_hqe_timestamp] 565 self._check_finished_job(created_timestamp, 566 hqe_list, 567 newer_hqe_timestamp) 568 self.mox.ResetAll() 569 hqe_list.reverse() 570 self._check_finished_job(created_timestamp, 571 hqe_list, 572 newer_hqe_timestamp) 573 574 575 def test_finished_regular_job_null_finished_times(self): 576 """Test getting the timestamp for an aborted regular job. 577 578 Tests the return value for 579 `RegularJobDirectory.get_timestamp_if_finished()` when 580 the AFE indicates the job is finished and the job has aborted host 581 queue entries. 582 583 """ 584 timestamp = _make_timestamp(0, True) 585 self._check_finished_job(timestamp, [], timestamp) 586 587 588 def test_unfinished_regular_job(self): 589 """Test getting the timestamp for an unfinished regular job. 590 591 Tests the return value for 592 `RegularJobDirectory.get_timestamp_if_finished()` when 593 the AFE indicates the job is not finished. 594 595 """ 596 job = job_directories.RegularJobDirectory('118-fubar') 597 job_directories._AFE.get_jobs( 598 id=job._id, finished=True).AndReturn([]) 599 self.mox.ReplayAll() 600 self.assertIsNone(job.get_timestamp_if_finished()) 601 self.mox.VerifyAll() 602 603 604 def test_finished_special_job(self): 605 """Test getting the timestamp for a finished special job. 606 607 Tests the return value for 608 `SpecialJobDirectory.get_timestamp_if_finished()` when 609 the AFE indicates the job is finished. 610 611 """ 612 job = job_directories.SpecialJobDirectory( 613 'hosts/host1/118-reset') 614 timestamp = _make_timestamp(0, True) 615 job_directories._AFE.get_special_tasks( 616 id=job._id, is_complete=True).AndReturn( 617 [_MockSpecialTask(timestamp)]) 618 self.mox.ReplayAll() 619 self.assertEqual(timestamp, 620 job.get_timestamp_if_finished()) 621 self.mox.VerifyAll() 622 623 624 def test_unfinished_special_job(self): 625 """Test getting the timestamp for an unfinished special job. 626 627 Tests the return value for 628 `SpecialJobDirectory.get_timestamp_if_finished()` when 629 the AFE indicates the job is not finished. 630 631 """ 632 job = job_directories.SpecialJobDirectory( 633 'hosts/host1/118-reset') 634 job_directories._AFE.get_special_tasks( 635 id=job._id, is_complete=True).AndReturn([]) 636 self.mox.ReplayAll() 637 self.assertIsNone(job.get_timestamp_if_finished()) 638 self.mox.VerifyAll() 639 640 641class _TempResultsDirTestBase(mox.MoxTestBase): 642 """Base class for tests using a temporary results directory.""" 643 644 REGULAR_JOBLIST = [ 645 '111-fubar', '112-fubar', '113-fubar', '114-snafu'] 646 HOST_LIST = ['host1', 'host2', 'host3'] 647 SPECIAL_JOBLIST = [ 648 'hosts/host1/333-reset', 'hosts/host1/334-reset', 649 'hosts/host2/444-reset', 'hosts/host3/555-reset'] 650 651 652 def setUp(self): 653 super(_TempResultsDirTestBase, self).setUp() 654 self._resultsroot = tempfile.mkdtemp() 655 self._cwd = os.getcwd() 656 os.chdir(self._resultsroot) 657 658 659 def tearDown(self): 660 os.chdir(self._cwd) 661 shutil.rmtree(self._resultsroot) 662 super(_TempResultsDirTestBase, self).tearDown() 663 664 665 def make_job(self, jobdir): 666 """Create a job with results in `self._resultsroot`. 667 668 @param jobdir Name of the subdirectory to be created in 669 `self._resultsroot`. 670 671 """ 672 os.mkdir(jobdir) 673 return _MockJobDirectory(jobdir) 674 675 676 def make_job_hierarchy(self): 677 """Create a sample hierarchy of job directories. 678 679 `self.REGULAR_JOBLIST` is a list of directories for regular 680 jobs to be created; `self.SPECIAL_JOBLIST` is a list of 681 directories for special jobs to be created. 682 683 """ 684 for d in self.REGULAR_JOBLIST: 685 os.mkdir(d) 686 hostsdir = 'hosts' 687 os.mkdir(hostsdir) 688 for host in self.HOST_LIST: 689 os.mkdir(os.path.join(hostsdir, host)) 690 for d in self.SPECIAL_JOBLIST: 691 os.mkdir(d) 692 693 694class OffloadDirectoryTests(_TempResultsDirTestBase): 695 """Tests for `offload_dir()`.""" 696 697 def setUp(self): 698 super(OffloadDirectoryTests, self).setUp() 699 # offload_dir() logs messages; silence them. 700 self._saved_loglevel = logging.getLogger().getEffectiveLevel() 701 logging.getLogger().setLevel(logging.CRITICAL+1) 702 self._job = self.make_job(self.REGULAR_JOBLIST[0]) 703 self.mox.StubOutWithMock(gs_offloader, 'get_cmd_list') 704 self.mox.StubOutWithMock(signal, 'alarm') 705 706 707 def tearDown(self): 708 logging.getLogger().setLevel(self._saved_loglevel) 709 super(OffloadDirectoryTests, self).tearDown() 710 711 712 def _mock_offload_dir_calls(self, command, queue_args): 713 """Mock out the calls needed by `offload_dir()`. 714 715 This covers only the calls made when there is no timeout. 716 717 @param command Command list to be returned by the mocked 718 call to `get_cmd_list()`. 719 720 """ 721 signal.alarm(gs_offloader.OFFLOAD_TIMEOUT_SECS) 722 command.append(queue_args[0]) 723 gs_offloader.get_cmd_list( 724 False, queue_args[0], 725 '%s%s' % (utils.DEFAULT_OFFLOAD_GSURI, 726 queue_args[1])).AndReturn(command) 727 signal.alarm(0) 728 signal.alarm(0) 729 730 731 def _run_offload_dir(self, should_succeed): 732 """Make one call to `offload_dir()`. 733 734 The caller ensures all mocks are set up already. 735 736 @param should_succeed True iff the call to `offload_dir()` 737 is expected to succeed and remove the 738 offloaded job directory. 739 740 """ 741 self.mox.ReplayAll() 742 gs_offloader.get_offload_dir_func( 743 utils.DEFAULT_OFFLOAD_GSURI, False)( 744 self._job.queue_args[0], 745 self._job.queue_args[1]) 746 self.mox.VerifyAll() 747 self.assertEqual(not should_succeed, 748 os.path.isdir(self._job.queue_args[0])) 749 750 751 def test_offload_success(self): 752 """Test that `offload_dir()` can succeed correctly.""" 753 self._mock_offload_dir_calls(['test', '-d'], 754 self._job.queue_args) 755 self._run_offload_dir(True) 756 757 758 def test_offload_failure(self): 759 """Test that `offload_dir()` can fail correctly.""" 760 self._mock_offload_dir_calls(['test', '!', '-d'], 761 self._job.queue_args) 762 self._run_offload_dir(False) 763 764 765 def test_offload_timeout_early(self): 766 """Test that `offload_dir()` times out correctly. 767 768 This test triggers timeout at the earliest possible moment, 769 at the first call to set the timeout alarm. 770 771 """ 772 signal.alarm(gs_offloader.OFFLOAD_TIMEOUT_SECS).AndRaise( 773 gs_offloader.TimeoutException('fubar')) 774 signal.alarm(0) 775 self._run_offload_dir(False) 776 777 778 def test_offload_timeout_late(self): 779 """Test that `offload_dir()` times out correctly. 780 781 This test triggers timeout at the latest possible moment, at 782 the call to clear the timeout alarm. 783 784 """ 785 signal.alarm(gs_offloader.OFFLOAD_TIMEOUT_SECS) 786 gs_offloader.get_cmd_list( 787 False, mox.IgnoreArg(), mox.IgnoreArg()).AndReturn( 788 ['test', '-d', self._job.queue_args[0]]) 789 signal.alarm(0).AndRaise( 790 gs_offloader.TimeoutException('fubar')) 791 signal.alarm(0) 792 self._run_offload_dir(False) 793 794 795 def test_sanitize_dir(self): 796 """Test that folder/file name with invalid character can be corrected. 797 """ 798 results_folder = tempfile.mkdtemp() 799 invalid_chars = '_'.join(gs_offloader.INVALID_GS_CHARS) 800 invalid_files = [] 801 invalid_folder = os.path.join( 802 results_folder, 803 'invalid_name_folder_%s' % invalid_chars) 804 invalid_files.append(os.path.join( 805 invalid_folder, 806 'invalid_name_file_%s' % invalid_chars)) 807 for r in gs_offloader.INVALID_GS_CHAR_RANGE: 808 for c in range(r[0], r[1]+1): 809 # NULL cannot be in file name. 810 if c != 0: 811 invalid_files.append(os.path.join( 812 invalid_folder, 813 'invalid_name_file_%s' % chr(c))) 814 good_folder = os.path.join(results_folder, 'valid_name_folder') 815 good_file = os.path.join(good_folder, 'valid_name_file') 816 for folder in [invalid_folder, good_folder]: 817 os.makedirs(folder) 818 for f in invalid_files + [good_file]: 819 with open(f, 'w'): 820 pass 821 gs_offloader.sanitize_dir(results_folder) 822 for _, dirs, files in os.walk(results_folder): 823 for name in dirs + files: 824 self.assertEqual(name, gs_offloader.get_sanitized_name(name)) 825 for c in name: 826 self.assertFalse(c in gs_offloader.INVALID_GS_CHARS) 827 for r in gs_offloader.INVALID_GS_CHAR_RANGE: 828 self.assertFalse(ord(c) >= r[0] and ord(c) <= r[1]) 829 self.assertTrue(os.path.exists(good_file)) 830 shutil.rmtree(results_folder) 831 832 833 def check_limit_file_count(self, is_test_job=True): 834 """Test that folder with too many files can be compressed. 835 836 @param is_test_job: True to check the method with test job result 837 folder. Set to False for special task folder. 838 """ 839 results_folder = tempfile.mkdtemp() 840 host_folder = os.path.join( 841 results_folder, 842 'lab1-host1' if is_test_job else 'hosts/lab1-host1/1-repair') 843 debug_folder = os.path.join(host_folder, 'debug') 844 sysinfo_folder = os.path.join(host_folder, 'sysinfo') 845 for folder in [debug_folder, sysinfo_folder]: 846 os.makedirs(folder) 847 for i in range(10): 848 with open(os.path.join(folder, str(i)), 'w') as f: 849 f.write('test') 850 851 gs_offloader.MAX_FILE_COUNT = 100 852 gs_offloader.limit_file_count( 853 results_folder if is_test_job else host_folder) 854 self.assertTrue(os.path.exists(sysinfo_folder)) 855 856 gs_offloader.MAX_FILE_COUNT = 10 857 gs_offloader.limit_file_count( 858 results_folder if is_test_job else host_folder) 859 self.assertFalse(os.path.exists(sysinfo_folder)) 860 self.assertTrue(os.path.exists(sysinfo_folder + '.tgz')) 861 self.assertTrue(os.path.exists(debug_folder)) 862 863 shutil.rmtree(results_folder) 864 865 866 def test_limit_file_count(self): 867 """Test that folder with too many files can be compressed. 868 """ 869 self.check_limit_file_count(is_test_job=True) 870 self.check_limit_file_count(is_test_job=False) 871 872 873class JobDirectoryOffloadTests(_TempResultsDirTestBase): 874 """Tests for `_JobDirectory.enqueue_offload()`. 875 876 When testing with a `days_old` parameter of 0, we use 877 `set_finished()` instead of `set_expired()`. This causes the 878 job's timestamp to be set in the future. This is done so as 879 to test that when `days_old` is 0, the job is always treated 880 as eligible for offload, regardless of the timestamp's value. 881 882 Testing covers the following assertions: 883 A. Each time `enqueue_offload()` is called, a message that 884 includes the job's directory name will be logged using 885 `logging.debug()`, regardless of whether the job was 886 enqueued. Nothing else is allowed to be logged. 887 B. If the job is not eligible to be offloaded, 888 `get_failure_time()` and `get_failure_count()` are 0. 889 C. If the job is not eligible for offload, nothing is 890 enqueued in `queue`. 891 D. When the job is offloaded, `get_failure_count()` increments 892 each time. 893 E. When the job is offloaded, the appropriate parameters are 894 enqueued exactly once. 895 F. The first time a job is offloaded, `get_failure_time()` is 896 set to the current time. 897 G. `get_failure_time()` only changes the first time that the 898 job is offloaded. 899 900 The test cases below are designed to exercise all of the 901 meaningful state transitions at least once. 902 903 """ 904 905 def setUp(self): 906 super(JobDirectoryOffloadTests, self).setUp() 907 self._job = self.make_job(self.REGULAR_JOBLIST[0]) 908 self._queue = Queue.Queue() 909 910 911 def _offload_unexpired_job(self, days_old): 912 """Make calls to `enqueue_offload()` for an unexpired job. 913 914 This method tests assertions B and C that calling 915 `enqueue_offload()` has no effect. 916 917 """ 918 self.assertEqual(self._job.get_failure_count(), 0) 919 self.assertEqual(self._job.get_failure_time(), 0) 920 self._job.enqueue_offload(self._queue, days_old) 921 self._job.enqueue_offload(self._queue, days_old) 922 self.assertTrue(self._queue.empty()) 923 self.assertEqual(self._job.get_failure_count(), 0) 924 self.assertEqual(self._job.get_failure_time(), 0) 925 self.assertFalse(self._job.is_reportable()) 926 927 928 def _offload_expired_once(self, days_old, count): 929 """Make one call to `enqueue_offload()` for an expired job. 930 931 This method tests assertions D and E regarding side-effects 932 expected when a job is offloaded. 933 934 """ 935 self._job.enqueue_offload(self._queue, days_old) 936 self.assertEqual(self._job.get_failure_count(), count) 937 self.assertFalse(self._queue.empty()) 938 v = self._queue.get_nowait() 939 self.assertTrue(self._queue.empty()) 940 self.assertEqual(v, self._job.queue_args) 941 942 943 def _offload_expired_job(self, days_old): 944 """Make calls to `enqueue_offload()` for a just-expired job. 945 946 This method directly tests assertions F and G regarding 947 side-effects on `get_failure_time()`. 948 949 """ 950 t0 = time.time() 951 self._offload_expired_once(days_old, 1) 952 self.assertFalse(self._job.is_reportable()) 953 t1 = self._job.get_failure_time() 954 self.assertLessEqual(t1, time.time()) 955 self.assertGreaterEqual(t1, t0) 956 self._offload_expired_once(days_old, 2) 957 self.assertTrue(self._job.is_reportable()) 958 self.assertEqual(self._job.get_failure_time(), t1) 959 self._offload_expired_once(days_old, 3) 960 self.assertTrue(self._job.is_reportable()) 961 self.assertEqual(self._job.get_failure_time(), t1) 962 963 964 def test_case_1_no_expiration(self): 965 """Test a series of `enqueue_offload()` calls with `days_old` of 0. 966 967 This tests that offload works as expected if calls are 968 made both before and after the job becomes expired. 969 970 """ 971 self._offload_unexpired_job(0) 972 self._job.set_finished(0) 973 self._offload_expired_job(0) 974 975 976 def test_case_2_no_expiration(self): 977 """Test a series of `enqueue_offload()` calls with `days_old` of 0. 978 979 This tests that offload works as expected if calls are made 980 only after the job becomes expired. 981 982 """ 983 self._job.set_finished(0) 984 self._offload_expired_job(0) 985 986 987 def test_case_1_with_expiration(self): 988 """Test a series of `enqueue_offload()` calls with `days_old` non-zero. 989 990 This tests that offload works as expected if calls are made 991 before the job finishes, before the job expires, and after 992 the job expires. 993 994 """ 995 self._offload_unexpired_job(_TEST_EXPIRATION_AGE) 996 self._job.set_finished(_TEST_EXPIRATION_AGE) 997 self._offload_unexpired_job(_TEST_EXPIRATION_AGE) 998 self._job.set_expired(_TEST_EXPIRATION_AGE) 999 self._offload_expired_job(_TEST_EXPIRATION_AGE) 1000 1001 1002 def test_case_2_with_expiration(self): 1003 """Test a series of `enqueue_offload()` calls with `days_old` non-zero. 1004 1005 This tests that offload works as expected if calls are made 1006 between finishing and expiration, and after the job expires. 1007 1008 """ 1009 self._job.set_finished(_TEST_EXPIRATION_AGE) 1010 self._offload_unexpired_job(_TEST_EXPIRATION_AGE) 1011 self._job.set_expired(_TEST_EXPIRATION_AGE) 1012 self._offload_expired_job(_TEST_EXPIRATION_AGE) 1013 1014 1015 def test_case_3_with_expiration(self): 1016 """Test a series of `enqueue_offload()` calls with `days_old` non-zero. 1017 1018 This tests that offload works as expected if calls are made 1019 only before finishing and after expiration. 1020 1021 """ 1022 self._offload_unexpired_job(_TEST_EXPIRATION_AGE) 1023 self._job.set_expired(_TEST_EXPIRATION_AGE) 1024 self._offload_expired_job(_TEST_EXPIRATION_AGE) 1025 1026 1027 def test_case_4_with_expiration(self): 1028 """Test a series of `enqueue_offload()` calls with `days_old` non-zero. 1029 1030 This tests that offload works as expected if calls are made 1031 only after expiration. 1032 1033 """ 1034 self._job.set_expired(_TEST_EXPIRATION_AGE) 1035 self._offload_expired_job(_TEST_EXPIRATION_AGE) 1036 1037 1038class GetJobDirectoriesTests(_TempResultsDirTestBase): 1039 """Tests for `_JobDirectory.get_job_directories()`.""" 1040 1041 def setUp(self): 1042 super(GetJobDirectoriesTests, self).setUp() 1043 self.make_job_hierarchy() 1044 os.mkdir('not-a-job') 1045 open('not-a-dir', 'w').close() 1046 1047 1048 def _run_get_directories(self, cls, expected_list): 1049 """Test `get_job_directories()` for the given class. 1050 1051 Calls the method, and asserts that the returned list of 1052 directories matches the expected return value. 1053 1054 @param expected_list Expected return value from the call. 1055 """ 1056 dirlist = cls.get_job_directories() 1057 self.assertEqual(set(dirlist), set(expected_list)) 1058 1059 1060 def test_get_regular_jobs(self): 1061 """Test `RegularJobDirectory.get_job_directories()`.""" 1062 self._run_get_directories(job_directories.RegularJobDirectory, 1063 self.REGULAR_JOBLIST) 1064 1065 1066 def test_get_special_jobs(self): 1067 """Test `SpecialJobDirectory.get_job_directories()`.""" 1068 self._run_get_directories(job_directories.SpecialJobDirectory, 1069 self.SPECIAL_JOBLIST) 1070 1071 1072class AddJobsTests(_TempResultsDirTestBase): 1073 """Tests for `Offloader._add_new_jobs()`.""" 1074 1075 MOREJOBS = ['115-fubar', '116-fubar', '117-fubar', '118-snafu'] 1076 1077 def setUp(self): 1078 super(AddJobsTests, self).setUp() 1079 self._initial_job_names = ( 1080 set(self.REGULAR_JOBLIST) | set(self.SPECIAL_JOBLIST)) 1081 self.make_job_hierarchy() 1082 self._offloader = gs_offloader.Offloader(_get_options(['-a'])) 1083 self.mox.StubOutWithMock(logging, 'debug') 1084 1085 1086 def _run_add_new_jobs(self, expected_key_set): 1087 """Basic test assertions for `_add_new_jobs()`. 1088 1089 Asserts the following: 1090 * The keys in the offloader's `_open_jobs` dictionary 1091 matches the expected set of keys. 1092 * For every job in `_open_jobs`, the job has the expected 1093 directory name. 1094 1095 """ 1096 count = len(expected_key_set) - len(self._offloader._open_jobs) 1097 logging.debug(mox.IgnoreArg(), count) 1098 self.mox.ReplayAll() 1099 self._offloader._add_new_jobs() 1100 self.assertEqual(expected_key_set, 1101 set(self._offloader._open_jobs.keys())) 1102 for jobkey, job in self._offloader._open_jobs.items(): 1103 self.assertEqual(jobkey, job._dirname) 1104 self.mox.VerifyAll() 1105 self.mox.ResetAll() 1106 1107 1108 def test_add_jobs_empty(self): 1109 """Test adding jobs to an empty dictionary. 1110 1111 Calls the offloader's `_add_new_jobs()`, then perform 1112 the assertions of `self._check_open_jobs()`. 1113 1114 """ 1115 self._run_add_new_jobs(self._initial_job_names) 1116 1117 1118 def test_add_jobs_non_empty(self): 1119 """Test adding jobs to a non-empty dictionary. 1120 1121 Calls the offloader's `_add_new_jobs()` twice; once from 1122 initial conditions, and then again after adding more 1123 directories. After the second call, perform the assertions 1124 of `self._check_open_jobs()`. Additionally, assert that 1125 keys added by the first call still map to their original 1126 job object after the second call. 1127 1128 """ 1129 self._run_add_new_jobs(self._initial_job_names) 1130 jobs_copy = self._offloader._open_jobs.copy() 1131 for d in self.MOREJOBS: 1132 os.mkdir(d) 1133 self._run_add_new_jobs(self._initial_job_names | 1134 set(self.MOREJOBS)) 1135 for key in jobs_copy.keys(): 1136 self.assertIs(jobs_copy[key], 1137 self._offloader._open_jobs[key]) 1138 1139 1140class JobStateTests(_TempResultsDirTestBase): 1141 """Tests for job state predicates. 1142 1143 This tests for the expected results from the 1144 `is_offloaded()` and `is_reportable()` predicate 1145 methods. 1146 1147 """ 1148 1149 def test_unfinished_job(self): 1150 """Test that an unfinished job reports the correct state. 1151 1152 A job is "unfinished" if it isn't marked complete in the 1153 database. A job in this state is neither "complete" nor 1154 "reportable". 1155 1156 """ 1157 job = self.make_job(self.REGULAR_JOBLIST[0]) 1158 self.assertFalse(job.is_offloaded()) 1159 self.assertFalse(job.is_reportable()) 1160 1161 1162 def test_incomplete_job(self): 1163 """Test that an incomplete job reports the correct state. 1164 1165 A job is "incomplete" if exactly one attempt has been made 1166 to offload the job, but its results directory still exists. 1167 A job in this state is neither "complete" nor "reportable". 1168 1169 """ 1170 job = self.make_job(self.REGULAR_JOBLIST[0]) 1171 job.set_incomplete() 1172 self.assertFalse(job.is_offloaded()) 1173 self.assertFalse(job.is_reportable()) 1174 1175 1176 def test_reportable_job(self): 1177 """Test that a reportable job reports the correct state. 1178 1179 A job is "reportable" if more than one attempt has been made 1180 to offload the job, and its results directory still exists. 1181 A job in this state is "reportable", but not "complete". 1182 1183 """ 1184 job = self.make_job(self.REGULAR_JOBLIST[0]) 1185 job.set_reportable() 1186 self.assertFalse(job.is_offloaded()) 1187 self.assertTrue(job.is_reportable()) 1188 1189 1190 def test_completed_job(self): 1191 """Test that a completed job reports the correct state. 1192 1193 A job is "completed" if at least one attempt has been made 1194 to offload the job, and its results directory still exists. 1195 A job in this state is "complete", and not "reportable". 1196 1197 """ 1198 job = self.make_job(self.REGULAR_JOBLIST[0]) 1199 job.set_complete() 1200 self.assertTrue(job.is_offloaded()) 1201 self.assertFalse(job.is_reportable()) 1202 1203 1204class ReportingTests(_TempResultsDirTestBase): 1205 """Tests for `Offloader._update_offload_results()`.""" 1206 1207 def setUp(self): 1208 super(ReportingTests, self).setUp() 1209 self._offloader = gs_offloader.Offloader(_get_options([])) 1210 self.mox.StubOutWithMock(email_manager.manager, 1211 'send_email') 1212 self.mox.StubOutWithMock(logging, 'debug') 1213 1214 1215 def _add_job(self, jobdir): 1216 """Add a job to the dictionary of unfinished jobs.""" 1217 j = self.make_job(jobdir) 1218 self._offloader._open_jobs[j._dirname] = j 1219 return j 1220 1221 1222 def _expect_log_message(self, new_open_jobs, with_failures): 1223 """Mock expected logging calls. 1224 1225 `_update_offload_results()` logs one message with the number 1226 of jobs removed from the open job set and the number of jobs 1227 still remaining. Additionally, if there are reportable 1228 jobs, then it logs the number of jobs that haven't yet 1229 offloaded. 1230 1231 This sets up the logging calls using `new_open_jobs` to 1232 figure the job counts. If `with_failures` is true, then 1233 the log message is set up assuming that all jobs in 1234 `new_open_jobs` have offload failures. 1235 1236 @param new_open_jobs New job set for calculating counts 1237 in the messages. 1238 @param with_failures Whether the log message with a 1239 failure count is expected. 1240 1241 """ 1242 count = len(self._offloader._open_jobs) - len(new_open_jobs) 1243 logging.debug(mox.IgnoreArg(), count, len(new_open_jobs)) 1244 if with_failures: 1245 logging.debug(mox.IgnoreArg(), len(new_open_jobs)) 1246 1247 1248 def _run_update_no_report(self, new_open_jobs): 1249 """Call `_update_offload_results()` expecting no report. 1250 1251 Initial conditions are set up by the caller. This calls 1252 `_update_offload_results()` once, and then checks these 1253 assertions: 1254 * The offloader's `_next_report_time` field is unchanged. 1255 * The offloader's new `_open_jobs` field contains only 1256 the entries in `new_open_jobs`. 1257 * The email_manager's `send_email` stub wasn't called. 1258 1259 @param new_open_jobs A dictionary representing the expected 1260 new value of the offloader's 1261 `_open_jobs` field. 1262 """ 1263 self.mox.ReplayAll() 1264 next_report_time = self._offloader._next_report_time 1265 self._offloader._update_offload_results() 1266 self.assertEqual(next_report_time, 1267 self._offloader._next_report_time) 1268 self.assertEqual(self._offloader._open_jobs, new_open_jobs) 1269 self.mox.VerifyAll() 1270 self.mox.ResetAll() 1271 1272 1273 def _run_update_with_report(self, new_open_jobs): 1274 """Call `_update_offload_results()` expecting an e-mail report. 1275 1276 Initial conditions are set up by the caller. This calls 1277 `_update_offload_results()` once, and then checks these 1278 assertions: 1279 * The offloader's `_next_report_time` field is updated 1280 to an appropriate new time. 1281 * The offloader's new `_open_jobs` field contains only 1282 the entries in `new_open_jobs`. 1283 * The email_manager's `send_email` stub was called. 1284 1285 @param new_open_jobs A dictionary representing the expected 1286 new value of the offloader's 1287 `_open_jobs` field. 1288 """ 1289 logging.debug(mox.IgnoreArg()) 1290 email_manager.manager.send_email( 1291 mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()) 1292 self.mox.ReplayAll() 1293 t0 = time.time() + gs_offloader.REPORT_INTERVAL_SECS 1294 self._offloader._update_offload_results() 1295 t1 = time.time() + gs_offloader.REPORT_INTERVAL_SECS 1296 next_report_time = self._offloader._next_report_time 1297 self.assertGreaterEqual(next_report_time, t0) 1298 self.assertLessEqual(next_report_time, t1) 1299 self.assertEqual(self._offloader._open_jobs, new_open_jobs) 1300 self.mox.VerifyAll() 1301 self.mox.ResetAll() 1302 1303 1304 def test_no_jobs(self): 1305 """Test `_update_offload_results()` with no open jobs. 1306 1307 Initial conditions are an empty `_open_jobs` list and 1308 `_next_report_time` in the past. Expected result is no 1309 e-mail report, and an empty `_open_jobs` list. 1310 1311 """ 1312 self._expect_log_message({}, False) 1313 self._run_update_no_report({}) 1314 1315 1316 def test_all_completed(self): 1317 """Test `_update_offload_results()` with only complete jobs. 1318 1319 Initial conditions are an `_open_jobs` list consisting of 1320 only completed jobs and `_next_report_time` in the past. 1321 Expected result is no e-mail report, and an empty 1322 `_open_jobs` list. 1323 1324 """ 1325 for d in self.REGULAR_JOBLIST: 1326 self._add_job(d).set_complete() 1327 self._expect_log_message({}, False) 1328 self._run_update_no_report({}) 1329 1330 1331 def test_none_finished(self): 1332 """Test `_update_offload_results()` with only unfinished jobs. 1333 1334 Initial conditions are an `_open_jobs` list consisting of 1335 only unfinished jobs and `_next_report_time` in the past. 1336 Expected result is no e-mail report, and no change to the 1337 `_open_jobs` list. 1338 1339 """ 1340 for d in self.REGULAR_JOBLIST: 1341 self._add_job(d) 1342 new_jobs = self._offloader._open_jobs.copy() 1343 self._expect_log_message(new_jobs, False) 1344 self._run_update_no_report(new_jobs) 1345 1346 1347 def test_none_reportable(self): 1348 """Test `_update_offload_results()` with only incomplete jobs. 1349 1350 Initial conditions are an `_open_jobs` list consisting of 1351 only incomplete jobs and `_next_report_time` in the past. 1352 Expected result is no e-mail report, and no change to the 1353 `_open_jobs` list. 1354 1355 """ 1356 for d in self.REGULAR_JOBLIST: 1357 self._add_job(d).set_incomplete() 1358 new_jobs = self._offloader._open_jobs.copy() 1359 self._expect_log_message(new_jobs, False) 1360 self._run_update_no_report(new_jobs) 1361 1362 1363 def test_report_not_ready(self): 1364 """Test `_update_offload_results()` e-mail throttling. 1365 1366 Initial conditions are an `_open_jobs` list consisting of 1367 only reportable jobs but with `_next_report_time` in 1368 the future. Expected result is no e-mail report, and no 1369 change to the `_open_jobs` list. 1370 1371 """ 1372 # N.B. This test may fail if its run time exceeds more than 1373 # about _MARGIN_SECS seconds. 1374 for d in self.REGULAR_JOBLIST: 1375 self._add_job(d).set_reportable() 1376 self._offloader._next_report_time += _MARGIN_SECS 1377 new_jobs = self._offloader._open_jobs.copy() 1378 self._expect_log_message(new_jobs, True) 1379 self._run_update_no_report(new_jobs) 1380 1381 1382 def test_reportable(self): 1383 """Test `_update_offload_results()` with reportable jobs. 1384 1385 Initial conditions are an `_open_jobs` list consisting of 1386 only reportable jobs and with `_next_report_time` in 1387 the past. Expected result is an e-mail report, and no 1388 change to the `_open_jobs` list. 1389 1390 """ 1391 for d in self.REGULAR_JOBLIST: 1392 self._add_job(d).set_reportable() 1393 new_jobs = self._offloader._open_jobs.copy() 1394 self._expect_log_message(new_jobs, True) 1395 self._run_update_with_report(new_jobs) 1396 1397 1398 def test_reportable_mixed(self): 1399 """Test `_update_offload_results()` with a mixture of jobs. 1400 1401 Initial conditions are an `_open_jobs` list consisting of 1402 one reportable jobs and the remainder of the jobs 1403 incomplete. The value of `_next_report_time` is in the 1404 past. Expected result is an e-mail report that includes 1405 both the reportable and the incomplete jobs, and no change 1406 to the `_open_jobs` list. 1407 1408 """ 1409 self._add_job(self.REGULAR_JOBLIST[0]).set_reportable() 1410 for d in self.REGULAR_JOBLIST[1:]: 1411 self._add_job(d).set_incomplete() 1412 new_jobs = self._offloader._open_jobs.copy() 1413 self._expect_log_message(new_jobs, True) 1414 self._run_update_with_report(new_jobs) 1415 1416 1417if __name__ == '__main__': 1418 unittest.main() 1419