1#!/usr/bin/python
2
3"""
4Selects all rows and columns that satisfy the condition specified
5and draws the matrix. There is a seperate SQL query made for every (x,y)
6in the matrix.
7"""
8
9print "Content-type: text/html\n"
10
11import sys, os, urllib, cgi, cgitb, re, datetime, time
12
13total_wall_time_start = time.time()
14
15import common
16from autotest_lib.tko import display, frontend, db, query_lib
17from autotest_lib.client.common_lib import kernel_versions
18
19html_header = """\
20<form action="/tko/compose_query.cgi" method="get">
21<table border="0">
22<tr>
23  <td>Column: </td>
24  <td>Row: </td>
25  <td>Condition: </td>
26  <td align="center">
27  <a href="http://autotest.kernel.org/wiki/AutotestTKOCondition">Help</a>
28  </td>
29</tr>
30<tr>
31  <td>
32  <SELECT NAME="columns">
33  %s
34 </SELECT>
35  </td>
36  <td>
37  <SELECT NAME="rows">
38  %s
39  </SELECT>
40  </td>
41  <td>
42    <input type="text" name="condition" size="30" value="%s">
43    <input type="hidden" name="title" value="%s">
44  </td>
45  <td align="center"><input type="submit" value="Submit">
46  </td>
47</tr>
48</table>
49</form>
50<form action="/tko/save_query.cgi" method="get">
51<table border="0">
52<tr>
53 <td>Name your query:&nbsp;&nbsp;</td>
54  <td>
55    <input type="text" name="label" size="15" value="">
56  </td>
57  <td align="center">&nbsp;<input type="submit" value="Save Query">
58  </td>
59 <td>&nbsp;&nbsp;<a href="/tko/query_history.cgi">View saved queries</a></td>
60  <td>
61    <input type="hidden" name="columns" value="%s">
62    <input type="hidden" name="rows" value="%s">
63    <input type="hidden" name="condition" value="%s">
64  </td>
65</tr>
66</table>
67</form>
68"""
69
70
71next_field = {
72    'machine_group': 'hostname',
73    'hostname': 'tag',
74    'tag': 'tag',
75
76    'kernel': 'test',
77    'test': 'label',
78    'label': 'tag',
79
80    'reason': 'tag',
81    'user': 'tag',
82    'status': 'tag',
83
84    'time': 'tag',
85    'time_daily': 'time',
86}
87
88
89def parse_field(form, form_field, field_default):
90    if not form_field in form:
91        return field_default
92    field_input = form[form_field].value.lower()
93    if field_input and field_input in frontend.test_view_field_dict:
94        return field_input
95    return field_default
96
97
98def parse_condition(form, form_field, field_default):
99    if not form_field in form:
100        return field_default
101    return form[form_field].value
102
103
104form = cgi.FieldStorage()
105
106title_field = parse_condition(form, 'title', '')
107row = parse_field(form, 'rows', 'kernel')
108column = parse_field(form, 'columns', 'machine_group')
109condition_field = parse_condition(form, 'condition', '')
110
111if 'brief' in form.keys() and form['brief'].value <> '0':
112    display.set_brief_mode()
113
114## caller can specify rows and columns that shall be included into the report
115## regardless of whether actual test data is available yet
116force_row_field = parse_condition(form,'force_row','')
117force_column_field = parse_condition(form,'force_column','')
118
119
120def split_forced_fields(force_field):
121    if force_field:
122        return force_field.split()
123    else:
124        return []
125
126force_row =  split_forced_fields(force_row_field)
127force_column =  split_forced_fields(force_column_field)
128
129cgitb.enable()
130db_obj = db.db()
131
132
133def construct_link(x, y):
134    next_row = row
135    next_column = column
136    condition_list = []
137    if condition_field != '':
138        condition_list.append(condition_field)
139    if y:
140        next_row = next_field[row]
141        condition_list.append("%s='%s'" % (row, y))
142    if x:
143        next_column = next_field[column]
144        condition_list.append("%s='%s'" % (column, x))
145    next_condition = '&'.join(condition_list)
146    link = '/tko/compose_query.cgi?' + urllib.urlencode({'columns': next_column,
147               'rows': next_row, 'condition': next_condition,
148               'title': title_field})
149    return link
150
151
152def construct_logs_link(x, y, job_tag):
153    job_path = frontend.html_root + job_tag + '/'
154    test = ''
155    if (row == 'test' and
156        not y.split('.')[0] in ('boot', 'build', 'install')):
157        test = y
158    if (column == 'test' and
159        not x.split('.')[0] in ('boot', 'build', 'install')):
160        test = x
161    return '/tko/retrieve_logs.cgi?' + urllib.urlencode({'job' : job_path,
162         'test' : test})
163
164
165def create_select_options(selected_val):
166    ret = ""
167    for option in sorted(frontend.test_view_field_dict.keys()):
168        if selected_val == option:
169            selected = " SELECTED"
170        else:
171            selected = ""
172
173        ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % \
174                        (option, selected, option)
175    return ret
176
177
178def map_kernel_base(kernel_name):
179    ## insert <br> after each / in kernel name
180    ## but spare consequtive //
181    kernel_name = kernel_name.replace('/','/<br>')
182    kernel_name = kernel_name.replace('/<br>/<br>','//')
183    return kernel_name
184
185
186def header_tuneup(field_name, header):
187        ## header tune up depends on particular field name and may include:
188        ## - breaking header into several strings if it is long url
189        ## - creating date from datetime stamp
190        ## - possibly, expect more various refinements for different fields
191        if field_name == 'kernel':
192                return  map_kernel_base(header)
193        else:
194                return header
195
196
197# Kernel name mappings -- the kernels table 'printable' field is
198# effectively a sortable identifier for the kernel It encodes the base
199# release which is used for overall sorting, plus where patches are
200# applied it adds an increasing pNNN patch combination identifier
201# (actually the kernel_idx for the entry).  This allows sorting
202# as normal by the base kernel version and then sub-sorting by the
203# "first time we saw" a patch combination which should keep them in
204# approximatly date order.  This patch identifier is not suitable
205# for display, so we have to map it to a suitable html fragment for
206# display.  This contains the kernel base version plus the truncated
207# names of all the patches,
208#
209#     2.6.24-mm1 p112
210#     +add-new-string-functions-
211#     +x86-amd-thermal-interrupt
212#
213# This mapping is produced when the first mapping is request, with
214# a single query over the patches table; the result is then cached.
215#
216# Note: that we only count a base version as patched if it contains
217# patches which are not already "expressed" in the base version.
218# This includes both -gitN and -mmN kernels.
219map_kernel_map = None
220
221
222def map_kernel_init():
223    fields = ['base', 'k.kernel_idx', 'name', 'url']
224    map = {}
225    for (base, idx, name, url) in db_obj.select(','.join(fields),
226            'tko_kernels k, tko_patches p', 'k.kernel_idx=p.kernel_idx'):
227        match = re.match(r'.*(-mm[0-9]+|-git[0-9]+)\.(bz2|gz)$', url)
228        if match:
229            continue
230
231        key = base + ' p%d' % (idx)
232        if not map.has_key(key):
233            map[key] = map_kernel_base(base) + ' p%d' % (idx)
234        map[key] += ('<br>+<span title="' + name + '">' +
235                 name[0:25] + '</span>')
236
237    return map
238
239
240def map_kernel(name):
241    global map_kernel_map
242    if map_kernel_map is None:
243        map_kernel_map = map_kernel_init()
244
245    if map_kernel_map.has_key(name):
246        return map_kernel_map[name]
247
248    return map_kernel_base(name.split(' ')[0])
249
250
251field_map = {
252    'kernel':map_kernel
253}
254
255sql_wall_time = 0
256
257def gen_matrix():
258    where = None
259    if condition_field.strip() != '':
260        try:
261            where = query_lib.parse_scrub_and_gen_condition(
262                condition_field, frontend.test_view_field_dict)
263            print "<!-- where clause: %s -->" % (where,)
264        except:
265            msg = "Unspecified error when parsing condition"
266            return [[display.box(msg)]]
267
268    wall_time_start = time.time()
269    try:
270        ## Unfortunately, we can not request reasons of failure always
271        ## because it may result in an inflated size of data transfer
272        ## (at the moment we fetch 500 bytes of reason descriptions into
273        ## each cell )
274        ## If 'status' in [row,column] then either width or height
275        ## of the table <=7, hence table is not really 2D, and
276        ## query_reason is relatively save.
277        ## At the same time view when either rows or columns grouped
278        ## by status is when users need reasons of failures the most.
279
280        ## TO DO: implement [Show/Hide reasons] button or link in
281        ## all views and make thorough performance testing
282        test_data = frontend.get_matrix_data(db_obj, column, row, where,
283                query_reasons = ('status' in [row,column])
284                )
285        global sql_wall_time
286        sql_wall_time = time.time() - wall_time_start
287
288    except db.MySQLTooManyRows, error:
289        return [[display.box(str(error))]]
290
291    for f_row in force_row:
292        if not f_row in test_data.y_values:
293            test_data.y_values.append(f_row)
294    for f_column in force_column:
295        if not f_column in test_data.x_values:
296            test_data.x_values.append(f_column)
297
298    if not test_data.y_values:
299        msg = "There are no results for this query (yet?)."
300        return [[display.box(msg)]]
301
302    dict_url = {'columns': row,
303               'rows': column, 'condition': condition_field,
304               'title': title_field}
305    link = '/tko/compose_query.cgi?' + urllib.urlencode(dict_url)
306    header_row = [display.box("<center>(Flip Axis)</center>", link=link)]
307
308    for x in test_data.x_values:
309        dx = x
310        if field_map.has_key(column):
311            dx = field_map[column](x)
312        x_header = header_tuneup(column, dx)
313        link = construct_link(x, None)
314        header_row.append(display.box(x_header,header=True,link=link))
315
316    matrix = [header_row]
317    # For each row, we are looping horizontally over the columns.
318    for y in test_data.y_values:
319        dy = y
320        if field_map.has_key(row):
321            dy = field_map[row](y)
322        y_header = header_tuneup(row, dy)
323        link = construct_link(None, y)
324        cur_row = [display.box(y_header, header=True, link=link)]
325        for x in test_data.x_values:
326            ## next 2 lines: temporary, until non timestamped
327            ## records are in the database
328            if x==datetime.datetime(1970,1,1): x = None
329            if y==datetime.datetime(1970,1,1): y = None
330            try:
331                box_data = test_data.data[x][y]
332            except:
333                cur_row.append(display.box(None, None,
334                           row_label=y, column_label=x))
335                continue
336            job_tag = test_data.data[x][y].job_tag
337            if job_tag:
338                link = construct_logs_link(x, y, job_tag)
339            else:
340                link = construct_link(x, y)
341
342            apnd = display.status_precounted_box(db_obj, box_data,
343                                 link, y, x)
344            cur_row.append(apnd)
345        matrix.append(cur_row)
346    return matrix
347
348
349def main():
350    if display.is_brief_mode():
351        ## create main grid table only as provided by gen_matrix()
352        display.print_table(gen_matrix())
353    else:
354        # create the actual page
355        print '<html><head><title>'
356        print 'Filtered Autotest Results'
357        print '</title></head><body>'
358        display.print_main_header()
359        print html_header % (create_select_options(column),
360                     create_select_options(row),
361                     condition_field, title_field,
362                     ## history form
363                     column,row,condition_field)
364        if title_field:
365            print '<h1> %s </h1>' % (title_field)
366        print display.color_keys_row()
367        display.print_table(gen_matrix())
368        print display.color_keys_row()
369        total_wall_time = time.time() - total_wall_time_start
370
371        perf_info = '<p style="font-size:x-small;">'
372        perf_info += 'sql access wall time = %s secs,' % sql_wall_time
373        perf_info += 'total wall time = %s secs</p>' % total_wall_time
374        print perf_info
375        print '</body></html>'
376
377
378main()
379