1#!/usr/bin/env python
2#
3# Copyright (C) 2018 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import datetime
19import logging
20import webapp2
21
22from google.appengine.ext import ndb
23
24from webapp.src import vtslab_status as Status
25from webapp.src.proto import model
26from webapp.src.utils import email_util
27from webapp.src.utils import logger
28from webapp.src.utils import model_util
29
30JOB_RESPONSE_TIMEOUT_SECONDS = 60 * 60
31
32
33class PeriodicJobHeartBeat(webapp2.RequestHandler):
34    """Main class for /tasks/job_heartbeat.
35
36    Used to find lost jobs and change their status properly.
37
38    Attributes:
39        logger: Logger class
40    """
41    logger = logger.Logger()
42
43    def get(self):
44        """Generates an HTML page based on the jobs kept in DB."""
45        self.logger.Clear()
46
47        job_query = model.JobModel.query(
48            model.JobModel.status == Status.JOB_STATUS_DICT["leased"])
49        jobs = job_query.fetch()
50
51        lost_jobs = []
52        for job in jobs:
53            if job.heartbeat_stamp:
54                job_timestamp = job.heartbeat_stamp
55            else:
56                job_timestamp = job.timestamp
57            if (datetime.datetime.now() -
58                    job_timestamp).seconds >= JOB_RESPONSE_TIMEOUT_SECONDS:
59                lost_jobs.append(job)
60
61        lost_jobs_to_put = []
62        devices_to_put = []
63        for job in lost_jobs:
64            self.logger.Println("Lost job found")
65            self.logger.Println("[hostname]{} [device]{} [test_name]{}".format(
66                job.hostname, job.device, job.test_name))
67            job.status = Status.JOB_STATUS_DICT["infra-err"]
68            lost_jobs_to_put.append(job)
69            model_util.UpdateParentSchedule(
70                job, Status.JOB_STATUS_DICT["infra-err"])
71
72            device_query = model.DeviceModel.query(
73                model.DeviceModel.serial.IN(job.serial))
74            devices = device_query.fetch()
75            for device in devices:
76                self.logger.Println("Device serial: {}".format(device.serial))
77                device.scheduling_status = Status.DEVICE_SCHEDULING_STATUS_DICT[
78                    "free"]
79                devices_to_put.append(device)
80
81        if lost_jobs_to_put:
82            ndb.put_multi(lost_jobs_to_put)
83            email_util.send_job_notification(lost_jobs_to_put)
84            self.logger.Println("{} jobs are updated.".format(
85                len(lost_jobs_to_put)))
86
87        if devices_to_put:
88            ndb.put_multi(devices_to_put)
89            self.logger.Println("{} devices are updated.".format(
90                len(devices_to_put)))
91
92        lines = self.logger.Get()
93        logging.info("\n".join([line.strip() for line in lines]))
94
95        self.response.write("<pre>\n" + "\n".join(lines) + "\n</pre>")
96