1"""Tests for job_directories."""
2
3from __future__ import absolute_import
4from __future__ import division
5from __future__ import print_function
6
7import contextlib
8import datetime
9import mox
10import os
11import shutil
12import tempfile
13import unittest
14
15import common
16from autotest_lib.site_utils import job_directories
17from autotest_lib.client.common_lib import time_utils
18
19
20class SwarmingJobDirectoryTestCase(unittest.TestCase):
21    """Tests SwarmingJobDirectory."""
22
23    def test_get_job_directories_legacy(self):
24        with _change_to_tempdir():
25            os.makedirs("swarming-3e4391423c3a4311/b")
26            os.mkdir("not-a-swarming-dir")
27            results = job_directories.SwarmingJobDirectory.get_job_directories()
28            self.assertEqual(set(results), {"swarming-3e4391423c3a4311"})
29
30    def test_get_job_directories(self):
31        with _change_to_tempdir():
32            os.makedirs("swarming-3e4391423c3a4310/1")
33            os.makedirs("swarming-3e4391423c3a4310/0")
34            open("swarming-3e4391423c3a4310/1/.ready_for_offload",
35                 'w+').close()
36            os.makedirs("swarming-3e4391423c3a4310/a")
37            open("swarming-3e4391423c3a4310/a/.ready_for_offload",
38                 'w+').close()
39            os.makedirs("swarming-34391423c3a4310/1/test_id")
40            os.makedirs("swarming-34391423c3a4310/1/test_id2")
41            open("swarming-34391423c3a4310/1/test_id/.ready_for_offload",
42                 'w+').close()
43            open("swarming-34391423c3a4310/1/test_id2/.ready_for_offload",
44                 'w+').close()
45            os.mkdir("not-a-swarming-dir")
46            results = job_directories.SwarmingJobDirectory.get_job_directories()
47            self.assertEqual(
48                    set(results), {
49                            "swarming-3e4391423c3a4310/1",
50                            "swarming-3e4391423c3a4310/a",
51                            "swarming-34391423c3a4310/1/test_id",
52                            "swarming-34391423c3a4310/1/test_id2"
53                    })
54
55
56class GetJobIDOrTaskID(unittest.TestCase):
57    """Tests get_job_id_or_task_id."""
58
59    def test_legacy_swarming_path(self):
60        self.assertEqual(
61                "3e4391423c3a4311",
62                job_directories.get_job_id_or_task_id(
63                        "/autotest/results/swarming-3e4391423c3a4311"),
64        )
65        self.assertEqual(
66                "3e4391423c3a4311",
67                job_directories.get_job_id_or_task_id(
68                        "swarming-3e4391423c3a4311"),
69        )
70
71    def test_swarming_path(self):
72        self.assertEqual(
73                "3e4391423c3a4311",
74                job_directories.get_job_id_or_task_id(
75                        "/autotest/results/swarming-3e4391423c3a4310/1"),
76        )
77        self.assertEqual(
78                "3e4391423c3a431f",
79                job_directories.get_job_id_or_task_id(
80                        "swarming-3e4391423c3a4310/f"),
81        )
82
83
84
85class JobDirectorySubclassTests(mox.MoxTestBase):
86    """Test specific to RegularJobDirectory and SpecialJobDirectory.
87
88    This provides coverage for the implementation in both
89    RegularJobDirectory and SpecialJobDirectory.
90
91    """
92
93    def setUp(self):
94        super(JobDirectorySubclassTests, self).setUp()
95        self.mox.StubOutWithMock(job_directories, '_AFE')
96
97
98    def test_regular_job_fields(self):
99        """Test the constructor for `RegularJobDirectory`.
100
101        Construct a regular job, and assert that the `dirname`
102        and `_id` attributes are set as expected.
103
104        """
105        resultsdir = '118-fubar'
106        job = job_directories.RegularJobDirectory(resultsdir)
107        self.assertEqual(job.dirname, resultsdir)
108        self.assertEqual(job._id, '118')
109
110
111    def test_special_job_fields(self):
112        """Test the constructor for `SpecialJobDirectory`.
113
114        Construct a special job, and assert that the `dirname`
115        and `_id` attributes are set as expected.
116
117        """
118        destdir = 'hosts/host1'
119        resultsdir = destdir + '/118-reset'
120        job = job_directories.SpecialJobDirectory(resultsdir)
121        self.assertEqual(job.dirname, resultsdir)
122        self.assertEqual(job._id, '118')
123
124
125    def _check_finished_job(self, jobtime, hqetimes, expected):
126        """Mock and test behavior of a finished job.
127
128        Initialize the mocks for a call to
129        `get_timestamp_if_finished()`, then simulate one call.
130        Assert that the returned timestamp matches the passed
131        in expected value.
132
133        @param jobtime Time used to construct a _MockJob object.
134        @param hqetimes List of times used to construct
135                        _MockHostQueueEntry objects.
136        @param expected Expected time to be returned by
137                        get_timestamp_if_finished
138
139        """
140        job = job_directories.RegularJobDirectory('118-fubar')
141        job_directories._AFE.get_jobs(
142                id=job._id, finished=True).AndReturn(
143                        [_MockJob(jobtime)])
144        job_directories._AFE.get_host_queue_entries(
145                finished_on__isnull=False,
146                job_id=job._id).AndReturn(
147                        [_MockHostQueueEntry(t) for t in hqetimes])
148        self.mox.ReplayAll()
149        self.assertEqual(expected, job.get_timestamp_if_finished())
150        self.mox.VerifyAll()
151
152
153    def test_finished_regular_job(self):
154        """Test getting the timestamp for a finished regular job.
155
156        Tests the return value for
157        `RegularJobDirectory.get_timestamp_if_finished()` when
158        the AFE indicates the job is finished.
159
160        """
161        created_timestamp = make_timestamp(1, True)
162        hqe_timestamp = make_timestamp(0, True)
163        self._check_finished_job(created_timestamp,
164                                 [hqe_timestamp],
165                                 hqe_timestamp)
166
167
168    def test_finished_regular_job_multiple_hqes(self):
169        """Test getting the timestamp for a regular job with multiple hqes.
170
171        Tests the return value for
172        `RegularJobDirectory.get_timestamp_if_finished()` when
173        the AFE indicates the job is finished and the job has multiple host
174        queue entries.
175
176        Tests that the returned timestamp is the latest timestamp in
177        the list of HQEs, regardless of the returned order.
178
179        """
180        created_timestamp = make_timestamp(2, True)
181        older_hqe_timestamp = make_timestamp(1, True)
182        newer_hqe_timestamp = make_timestamp(0, True)
183        hqe_list = [older_hqe_timestamp,
184                    newer_hqe_timestamp]
185        self._check_finished_job(created_timestamp,
186                                 hqe_list,
187                                 newer_hqe_timestamp)
188        self.mox.ResetAll()
189        hqe_list.reverse()
190        self._check_finished_job(created_timestamp,
191                                 hqe_list,
192                                 newer_hqe_timestamp)
193
194
195    def test_finished_regular_job_null_finished_times(self):
196        """Test getting the timestamp for an aborted regular job.
197
198        Tests the return value for
199        `RegularJobDirectory.get_timestamp_if_finished()` when
200        the AFE indicates the job is finished and the job has aborted host
201        queue entries.
202
203        """
204        timestamp = make_timestamp(0, True)
205        self._check_finished_job(timestamp, [], timestamp)
206
207
208    def test_unfinished_regular_job(self):
209        """Test getting the timestamp for an unfinished regular job.
210
211        Tests the return value for
212        `RegularJobDirectory.get_timestamp_if_finished()` when
213        the AFE indicates the job is not finished.
214
215        """
216        job = job_directories.RegularJobDirectory('118-fubar')
217        job_directories._AFE.get_jobs(
218                id=job._id, finished=True).AndReturn([])
219        self.mox.ReplayAll()
220        self.assertIsNone(job.get_timestamp_if_finished())
221        self.mox.VerifyAll()
222
223
224    def test_finished_special_job(self):
225        """Test getting the timestamp for a finished special job.
226
227        Tests the return value for
228        `SpecialJobDirectory.get_timestamp_if_finished()` when
229        the AFE indicates the job is finished.
230
231        """
232        job = job_directories.SpecialJobDirectory(
233                'hosts/host1/118-reset')
234        timestamp = make_timestamp(0, True)
235        job_directories._AFE.get_special_tasks(
236                id=job._id, is_complete=True).AndReturn(
237                    [_MockSpecialTask(timestamp)])
238        self.mox.ReplayAll()
239        self.assertEqual(timestamp,
240                         job.get_timestamp_if_finished())
241        self.mox.VerifyAll()
242
243
244    def test_unfinished_special_job(self):
245        """Test getting the timestamp for an unfinished special job.
246
247        Tests the return value for
248        `SpecialJobDirectory.get_timestamp_if_finished()` when
249        the AFE indicates the job is not finished.
250
251        """
252        job = job_directories.SpecialJobDirectory(
253                'hosts/host1/118-reset')
254        job_directories._AFE.get_special_tasks(
255                id=job._id, is_complete=True).AndReturn([])
256        self.mox.ReplayAll()
257        self.assertIsNone(job.get_timestamp_if_finished())
258        self.mox.VerifyAll()
259
260
261class JobExpirationTests(unittest.TestCase):
262    """Tests to exercise `job_directories.is_job_expired()`."""
263
264    def test_expired(self):
265        """Test detection of an expired job."""
266        timestamp = make_timestamp(_TEST_EXPIRATION_AGE, True)
267        self.assertTrue(
268            job_directories.is_job_expired(
269                _TEST_EXPIRATION_AGE, timestamp))
270
271
272    def test_alive(self):
273        """Test detection of a job that's not expired."""
274        # N.B.  This test may fail if its run time exceeds more than
275        # about _MARGIN_SECS seconds.
276        timestamp = make_timestamp(_TEST_EXPIRATION_AGE, False)
277        self.assertFalse(
278            job_directories.is_job_expired(
279                _TEST_EXPIRATION_AGE, timestamp))
280
281
282# When constructing sample time values for testing expiration,
283# allow this many seconds between the expiration time and the
284# current time.
285_MARGIN_SECS = 10.0
286# Test value to use for `days_old`, if nothing else is required.
287_TEST_EXPIRATION_AGE = 7
288
289
290class _MockJob(object):
291    """Class to mock the return value of `AFE.get_jobs()`."""
292    def __init__(self, created):
293        self.created_on = created
294
295
296class _MockHostQueueEntry(object):
297    """Class to mock the return value of `AFE.get_host_queue_entries()`."""
298    def __init__(self, finished):
299        self.finished_on = finished
300
301
302class _MockSpecialTask(object):
303    """Class to mock the return value of `AFE.get_special_tasks()`."""
304    def __init__(self, finished):
305        self.time_finished = finished
306
307
308@contextlib.contextmanager
309def _change_to_tempdir():
310    old_dir = os.getcwd()
311    tempdir = tempfile.mkdtemp('job_directories_unittest')
312    try:
313        os.chdir(tempdir)
314        yield
315    finally:
316        os.chdir(old_dir)
317        shutil.rmtree(tempdir)
318
319
320def make_timestamp(age_limit, is_expired):
321    """Create a timestamp for use by `job_directories.is_job_expired()`.
322
323    The timestamp will meet the syntactic requirements for
324    timestamps used as input to `is_job_expired()`.  If
325    `is_expired` is true, the timestamp will be older than
326    `age_limit` days before the current time; otherwise, the
327    date will be younger.
328
329    @param age_limit    The number of days before expiration of the
330                        target timestamp.
331    @param is_expired   Whether the timestamp should be expired
332                        relative to `age_limit`.
333
334    """
335    seconds = -_MARGIN_SECS
336    if is_expired:
337        seconds = -seconds
338    delta = datetime.timedelta(days=age_limit, seconds=seconds)
339    reference_time = datetime.datetime.now() - delta
340    return reference_time.strftime(time_utils.TIME_FMT)
341
342
343if __name__ == '__main__':
344    unittest.main()
345