1import os, re, db, sys, datetime
2import common
3from autotest_lib.client.common_lib import kernel_versions
4
5MAX_RECORDS = 50000L
6MAX_CELLS = 500000L
7
8tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
9root_url_file = os.path.join(tko, '.root_url')
10if os.path.exists(root_url_file):
11    html_root = open(root_url_file, 'r').readline().rstrip()
12else:
13    html_root = '/results/'
14
15
16class status_cell:
17    # One cell in the matrix of status data.
18    def __init__(self):
19        # Count is a dictionary: status -> count of tests with status
20        self.status_count = {}
21        self.reasons_list = []
22        self.job_tag = None
23        self.job_tag_count = 0
24
25
26    def add(self, status, count, job_tags, reasons = None):
27        assert count > 0
28
29        self.job_tag = job_tags
30        self.job_tag_count += count
31        if self.job_tag_count > 1:
32            self.job_tag = None
33
34        self.status_count[status] = count
35        ### status == 6 means 'GOOD'
36        if status != 6:
37            ## None implies sorting problems and extra CRs in a cell
38            if reasons:
39                self.reasons_list.append(reasons)
40
41
42class status_data:
43    def __init__(self, sql_rows, x_field, y_field, query_reasons = False):
44        data = {}
45        y_values = set()
46
47        # Walk through the query, filing all results by x, y info
48        for row in sql_rows:
49            if query_reasons:
50                (x,y, status, count, job_tags, reasons) = row
51            else:
52                (x,y, status, count, job_tags) = row
53                reasons = None
54            if not data.has_key(x):
55                data[x] = {}
56            if not data[x].has_key(y):
57                y_values.add(y)
58                data[x][y] = status_cell()
59            data[x][y].add(status, count, job_tags, reasons)
60
61        # 2-d hash of data - [x-value][y-value]
62        self.data = data
63        # List of possible columns (x-values)
64        self.x_values = smart_sort(data.keys(), x_field)
65        # List of rows columns (y-values)
66        self.y_values = smart_sort(list(y_values), y_field)
67        nCells = len(self.y_values)*len(self.x_values)
68        if nCells > MAX_CELLS:
69            msg = 'Exceeded allowed number of cells in a table'
70            raise db.MySQLTooManyRows(msg)
71
72
73def get_matrix_data(db_obj, x_axis, y_axis, where = None,
74                    query_reasons = False):
75    # Searches on the test_view table - x_axis and y_axis must both be
76    # column names in that table.
77    x_field = test_view_field_dict[x_axis]
78    y_field = test_view_field_dict[y_axis]
79    query_fields_list = [x_field, y_field, 'status','COUNT(status)']
80    query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)")
81    if query_reasons:
82        query_fields_list.append(
83                "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)"
84                )
85    fields = ','.join(query_fields_list)
86
87    group_by = '%s, %s, status' % (x_field, y_field)
88    rows = db_obj.select(fields, 'tko_test_view',
89                    where=where, group_by=group_by, max_rows = MAX_RECORDS)
90    return status_data(rows, x_field, y_field, query_reasons)
91
92
93# Dictionary used simply for fast lookups from short reference names for users
94# to fieldnames in test_view
95test_view_field_dict = {
96        'kernel'        : 'kernel_printable',
97        'hostname'      : 'machine_hostname',
98        'test'          : 'test',
99        'label'         : 'job_label',
100        'machine_group' : 'machine_group',
101        'reason'        : 'reason',
102        'tag'           : 'job_tag',
103        'user'          : 'job_username',
104        'status'        : 'status_word',
105        'time'          : 'test_finished_time',
106        'start_time'    : 'test_started_time',
107        'time_daily'    : 'DATE(test_finished_time)'
108}
109
110
111def smart_sort(list, field):
112    if field == 'kernel_printable':
113        def kernel_encode(kernel):
114            return kernel_versions.version_encode(kernel)
115        list.sort(key = kernel_encode, reverse = True)
116        return list
117    ## old records may contain time=None
118    ## make None comparable with timestamp datetime or date
119    elif field == 'test_finished_time':
120        def convert_None_to_datetime(date_time):
121            if not date_time:
122                return datetime.datetime(1970, 1, 1, 0, 0, 0)
123            else:
124                return date_time
125        list = map(convert_None_to_datetime, list)
126    elif field == 'DATE(test_finished_time)':
127        def convert_None_to_date(date):
128            if not date:
129                return datetime.date(1970, 1, 1)
130            else:
131                return date
132        list = map(convert_None_to_date, list)
133    list.sort()
134    return list
135
136
137class group:
138    @classmethod
139    def select(klass, db):
140        """Return all possible machine groups"""
141        rows = db.select('distinct machine_group', 'tko_machines',
142                                        'machine_group is not null')
143        groupnames = sorted([row[0] for row in rows])
144        return [klass(db, groupname) for groupname in groupnames]
145
146
147    def __init__(self, db, name):
148        self.name = name
149        self.db = db
150
151
152    def machines(self):
153        return machine.select(self.db, { 'machine_group' : self.name })
154
155
156    def tests(self, where = {}):
157        values = [self.name]
158        sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx'
159        sql += ' where m.machine_group=%s'
160        for key in where.keys():
161            sql += ' and %s=%%s' % key
162            values.append(where[key])
163        return test.select_sql(self.db, sql, values)
164
165
166class machine:
167    @classmethod
168    def select(klass, db, where = {}):
169        fields = ['machine_idx', 'hostname', 'machine_group', 'owner']
170        machines = []
171        for row in db.select(','.join(fields), 'tko_machines', where):
172            machines.append(klass(db, *row))
173        return machines
174
175
176    def __init__(self, db, idx, hostname, group, owner):
177        self.db = db
178        self.idx = idx
179        self.hostname = hostname
180        self.group = group
181        self.owner = owner
182
183
184class kernel:
185    @classmethod
186    def select(klass, db, where = {}):
187        fields = ['kernel_idx', 'kernel_hash', 'base', 'printable']
188        rows = db.select(','.join(fields), 'tko_kernels', where)
189        return [klass(db, *row) for row in rows]
190
191
192    def __init__(self, db, idx, hash, base, printable):
193        self.db = db
194        self.idx = idx
195        self.hash = hash
196        self.base = base
197        self.printable = printable
198        self.patches = []    # THIS SHOULD PULL IN PATCHES!
199
200
201class test:
202    @classmethod
203    def select(klass, db, where={}, distinct=False):
204        fields = ['test_idx', 'job_idx', 'test', 'subdir',
205                  'kernel_idx', 'status', 'reason', 'machine_idx']
206        tests = []
207        for row in db.select(','.join(fields), 'tko_tests', where,
208                             distinct):
209            tests.append(klass(db, *row))
210        return tests
211
212
213    @classmethod
214    def select_sql(klass, db, sql, values):
215        fields = ['test_idx', 'job_idx', 'test', 'subdir',
216                  'kernel_idx', 'status', 'reason', 'machine_idx']
217        fields = ['t.'+field for field in fields]
218        rows = db.select_sql(','.join(fields), 'tko_tests', sql, values)
219        return [klass(db, *row) for row in rows]
220
221
222    def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx,
223                 status_num, reason, machine_idx):
224        self.idx = test_idx
225        self.job = job(db, job_idx)
226        self.testname = testname
227        self.subdir = subdir
228        self.kernel_idx = kernel_idx
229        self.__kernel = None
230        self.__iterations = None
231        self.machine_idx = machine_idx
232        self.__machine = None
233        self.status_num = status_num
234        self.status_word = db.status_word[status_num]
235        self.reason = reason
236        self.db = db
237        if self.subdir:
238            self.url = html_root + self.job.tag + '/' + self.subdir
239        else:
240            self.url = None
241
242
243    def iterations(self):
244        """
245        Caching function for iterations
246        """
247        if not self.__iterations:
248            self.__iterations = {}
249            # A dictionary - dict{key} = [value1, value2, ....]
250            where = {'test_idx' : self.idx}
251            for i in iteration.select(self.db, where):
252                if self.__iterations.has_key(i.key):
253                    self.__iterations[i.key].append(i.value)
254                else:
255                    self.__iterations[i.key] = [i.value]
256        return self.__iterations
257
258
259    def kernel(self):
260        """
261        Caching function for kernels
262        """
263        if not self.__kernel:
264            where = {'kernel_idx' : self.kernel_idx}
265            self.__kernel = kernel.select(self.db, where)[0]
266        return self.__kernel
267
268
269    def machine(self):
270        """
271        Caching function for kernels
272        """
273        if not self.__machine:
274            where = {'machine_idx' : self.machine_idx}
275            self.__machine = machine.select(self.db, where)[0]
276        return self.__machine
277
278
279class job:
280    def __init__(self, db, job_idx):
281        where = {'job_idx' : job_idx}
282        rows = db.select('tag, machine_idx', 'tko_jobs', where)
283        if rows:
284            self.tag, self.machine_idx = rows[0]
285            self.job_idx = job_idx
286
287
288class iteration:
289    @classmethod
290    def select(klass, db, where):
291        fields = ['iteration', 'attribute', 'value']
292        iterations = []
293        rows = db.select(','.join(fields), 'tko_iteration_result', where)
294        for row in rows:
295            iterations.append(klass(*row))
296        return iterations
297
298
299    def __init__(self, iteration, key, value):
300        self.iteration = iteration
301        self.key = key
302        self.value = value
303
304# class patch:
305#       def __init__(self):
306#               self.spec = None
307