1#!/usr/bin/env python
2
3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""This script is to be run daily to report machine utilization stats across
8each board and pool.
9"""
10
11
12import argparse
13from datetime import date
14from datetime import datetime
15from datetime import timedelta
16
17import common
18from autotest_lib.client.common_lib import time_utils
19from autotest_lib.client.common_lib import utils
20from autotest_lib.site_utils import gmail_lib
21from autotest_lib.site_utils import host_history
22from autotest_lib.site_utils import host_history_utils
23from autotest_lib.site_utils import host_label_utils
24
25try:
26    from chromite.lib import metrics
27    from chromite.lib import ts_mon_config
28except ImportError:
29    metrics = utils.metrics_mock
30    ts_mon_config = utils.metrics_mock
31
32
33_MACHINE_UTILIZATION_RATE_HOURLY = metrics.Float(
34        'chromeos/autotest/host/machine_utilization_rate/hourly')
35_MACHINE_AVAILABILITY_RATE_HOURLY = metrics.Float(
36        'chromeos/autotest/host/machine_availability_rate/hourly')
37_MACHINE_IDLE_RATE_HOURLY = metrics.Float(
38        'chromeos/autotest/host/machine_idle_rate/hourly')
39_MACHINE_UTILIZATION_RATE_DAILY = metrics.Float(
40        'chromeos/autotest/host/machine_utilization_rate/daily')
41_MACHINE_AVAILABILITY_RATE_DAILY = metrics.Float(
42        'chromeos/autotest/host/machine_availability_rate/daily')
43_MACHINE_IDLE_RATE_DAILY = metrics.Float(
44        'chromeos/autotest/host/machine_idle_rate/daily')
45
46def report_stats(board, pool, start_time, end_time, span):
47    """Report machine stats for given board, pool and time period.
48
49    @param board: Name of board.
50    @param pool: Name of pool.
51    @param start_time: start time to collect stats.
52    @param end_time: end time to collect stats.
53    @param span: Number of hours that the stats should be collected for.
54    @return: Error message collected when calculating the stats.
55    """
56    print '================ %-12s %-12s ================' % (board, pool)
57    try:
58        history = host_history.get_history_details(start_time=start_time,
59                                                   end_time=end_time,
60                                                   board=board,
61                                                   pool=pool)
62    except host_history_utils.NoHostFoundException as e:
63        print 'No history found. Error:\n%s' % e
64        history = None
65        mur = -1
66        mar = -1
67        mir = -1
68
69    if history:
70        status_intervals = host_history_utils.get_status_intervals(history)
71        stats_all, num_hosts = host_history_utils.aggregate_hosts(
72                status_intervals)
73        total = 0
74        total_time = span*3600*num_hosts
75        for status, interval in stats_all.iteritems():
76            total += interval
77        if abs(total - total_time) > 10:
78            error = ('Status intervals do not add up. No stats will be '
79                     'collected for board: %s, pool: %s, diff: %s' %
80                     (board, pool, total - total_time))
81            hosts = []
82            for history_for_host in status_intervals:
83                total = 0
84                for interval in history_for_host.keys():
85                    total += interval[1] - interval[0]
86                if total > span*3600:
87                    hosts.append(history_for_host.values()[0]['metadata']['hostname'])
88            error += ' hosts: %s' % ','.join(hosts)
89            print error
90            return error
91
92        mur = host_history_utils.get_machine_utilization_rate(stats_all)
93        mar = host_history_utils.get_machine_availability_rate(stats_all)
94        mir = mar - mur
95
96        for status, interval in stats_all.iteritems():
97            print '%-18s %-16s %-10.2f%%' % (status, interval,
98                                             100*interval/total_time)
99        print 'Machine utilization rate  = %-4.2f%%' % (100*mur)
100        print 'Machine availability rate = %-4.2f%%' % (100*mar)
101
102    fields = {'board': board,
103              'pool': pool}
104    if span == 1:
105        _MACHINE_UTILIZATION_RATE_HOURLY.set(mur, fields=fields)
106        _MACHINE_AVAILABILITY_RATE_HOURLY.set(mar, fields=fields)
107        _MACHINE_IDLE_RATE_HOURLY.set(mir, fields=fields)
108    elif span == 24:
109        _MACHINE_UTILIZATION_RATE_DAILY.set(mur, fields=fields)
110        _MACHINE_AVAILABILITY_RATE_DAILY.set(mar, fields=fields)
111        _MACHINE_IDLE_RATE_DAILY.set(mir, fields=fields)
112
113
114def main():
115    """main script. """
116    parser = argparse.ArgumentParser()
117    parser.add_argument('--span', type=int, dest='span', default=1,
118                        help=('Number of hours that stats should be collected. '
119                              'If it is set to 24, the end time of stats being '
120                              'collected will set to the mid of the night. '
121                              'Default is set to 1 hour.'))
122    parser.add_argument('-e', '--email', dest='email', default=None,
123                        help='Email any errors to the given email address.')
124    options = parser.parse_args()
125
126    boards = host_label_utils.get_all_boards()
127    pools = ['bvt', 'suites', 'cq']
128
129    if options.span == 24:
130        today = datetime.combine(date.today(), datetime.min.time())
131        end_time = time_utils.to_epoch_time(today)
132    else:
133        now = datetime.now()
134        end_time = datetime(year=now.year, month=now.month, day=now.day,
135                            hour=now.hour)
136        end_time = time_utils.to_epoch_time(end_time)
137
138    start_time = end_time - timedelta(hours=options.span).total_seconds()
139    print ('Collecting host stats from %s to %s...' %
140           (time_utils.epoch_time_to_date_string(start_time),
141            time_utils.epoch_time_to_date_string(end_time)))
142
143    ts_mon_config.SetupTsMonGlobalState('collect_host_stats')
144
145    errors = []
146    if not boards:
147        errors.append('Error! No board found in metadb.')
148    for board in boards:
149        for pool in pools:
150            error = report_stats(board, pool, start_time, end_time,
151                                 options.span)
152            if error:
153                errors.append(error)
154    if options.email and errors:
155        gmail_lib.send_email(options.email,
156                             'Error occured when collecting host stats.',
157                             '\n'.join(errors))
158
159
160if __name__ == '__main__':
161    main()
162