1# Copyright 2018 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Library for frontend.afe.models.JobHandoff and job cleanup."""
6
7from __future__ import absolute_import
8from __future__ import division
9from __future__ import print_function
10
11import datetime
12import logging
13import socket
14
15from lucifer import autotest
16
17logger = logging.getLogger(__name__)
18
19
20_JOB_GRACE_SECS = 10
21
22
23def incomplete():
24    """Return a QuerySet of incomplete JobHandoffs.
25
26    JobHandoff created within a cutoff period are exempt to allow the
27    job the chance to acquire its lease file; otherwise, incomplete jobs
28    without an active lease are considered dead.
29
30    @returns: Django QuerySet
31    """
32    models = autotest.load('frontend.afe.models')
33    Q = autotest.deps_load('django.db.models').Q
34    # Time ---*---------|---------*-------|--->
35    #    incomplete   cutoff   newborn   now
36    cutoff = (datetime.datetime.now()
37              - datetime.timedelta(seconds=_JOB_GRACE_SECS))
38    return (models.JobHandoff.objects
39            .filter(completed=False, created__lt=cutoff)
40            .filter(Q(drone=socket.gethostname()) | Q(drone=None)))
41
42
43def clean_up(job_ids):
44    """Clean up failed jobs failed in database.
45
46    This brings the database into a clean state, which includes marking
47    the job, HQEs, and hosts.
48    """
49    if not job_ids:
50        return
51    models = autotest.load('frontend.afe.models')
52    logger.info('Cleaning up failed jobs: %r', job_ids)
53    hqes = models.HostQueueEntry.objects.filter(job_id__in=job_ids)
54    logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
55    _clean_up_hqes(hqes)
56    host_ids = {id for id in hqes.values_list('host_id', flat=True)
57                if id is not None}
58    logger.debug('Found Hosts associated with jobs: %r', host_ids)
59    _clean_up_hosts(host_ids)
60
61
62def _clean_up_hqes(hqes):
63    models = autotest.load('frontend.afe.models')
64    logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
65    hqes.update(complete=True,
66                active=False,
67                status=models.HostQueueEntry.Status.FAILED)
68    (hqes.exclude(started_on=None)
69     .update(finished_on=datetime.datetime.now()))
70
71
72def _clean_up_hosts(host_ids):
73    models = autotest.load('frontend.afe.models')
74    transaction = autotest.deps_load('django.db.transaction')
75    with transaction.commit_on_success():
76        active_hosts = {
77            id for id in (models.HostQueueEntry.objects
78                          .filter(active=True, complete=False)
79                          .values_list('host_id', flat=True))
80            if id is not None}
81        logger.debug('Found active Hosts: %r', active_hosts)
82        (models.Host.objects
83         .filter(id__in=host_ids)
84         .exclude(id__in=active_hosts)
85         .update(status=models.Host.Status.READY))
86
87
88def mark_complete(job_ids):
89    """Mark the corresponding JobHandoffs as completed."""
90    if not job_ids:
91        return
92    models = autotest.load('frontend.afe.models')
93    logger.info('Marking job handoffs complete: %r', job_ids)
94    (models.JobHandoff.objects
95     .filter(job_id__in=job_ids)
96     .update(completed=True))
97