1# Lint as: python2, python3
2# Copyright 2017 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7This is a utility to build an html page based on the directory summaries
8collected during the test.
9"""
10
11import os
12import re
13
14import common
15from autotest_lib.client.bin.result_tools import utils_lib
16from autotest_lib.client.common_lib import global_config
17
18
19CONFIG = global_config.global_config
20# Base url to open a file from Google Storage
21GS_FILE_BASE_URL = CONFIG.get_config_value('CROS', 'gs_file_base_url')
22
23# Default width of `size_trimmed_width`. If throttle is not applied, the block
24# of `size_trimmed_width` will be set to minimum to make the view more compact.
25DEFAULT_SIZE_TRIMMED_WIDTH = 50
26
27DEFAULT_RESULT_SUMMARY_NAME = 'result_summary.html'
28
29DIR_SUMMARY_PATTERN = 'dir_summary_\d+.json'
30
31# ==================================================
32# Following are key names used in the html templates:
33
34CSS = 'css'
35DIRS = 'dirs'
36GS_FILE_BASE_URL_KEY = 'gs_file_base_url'
37INDENTATION_KEY = 'indentation'
38JAVASCRIPT = 'javascript'
39JOB_DIR = 'job_dir'
40NAME = 'name'
41PATH = 'path'
42
43SIZE_CLIENT_COLLECTED = 'size_client_collected'
44
45SIZE_INFO = 'size_info'
46SIZE_ORIGINAL = 'size_original'
47SIZE_PERCENT = 'size_percent'
48SIZE_PERCENT_CLASS = 'size_percent_class'
49SIZE_PERCENT_CLASS_REGULAR = 'size_percent'
50SIZE_PERCENT_CLASS_TOP = 'top_size_percent'
51SIZE_SUMMARY = 'size_summary'
52SIZE_TRIMMED = 'size_trimmed'
53
54# Width of `size_trimmed` block`
55SIZE_TRIMMED_WIDTH = 'size_trimmed_width'
56
57SUBDIRS = 'subdirs'
58SUMMARY_TREE = 'summary_tree'
59# ==================================================
60
61# Text to show when test result is not throttled.
62NOT_THROTTLED = '(Not throttled)'
63
64
65PAGE_TEMPLATE = """
66<!DOCTYPE html>
67  <html>
68    <body onload="init()">
69      <h3>Summary of test results</h3>
70%(size_summary)s
71      <p>
72      <b>
73        Display format of a file or directory:
74      </b>
75      </p>
76      <p>
77        <span class="size_percent" style="width:auto">
78          [percentage of size in the parent directory]
79        </span>
80        <span class="size_original" style="width:auto">
81          [original size]
82        </span>
83        <span class="size_trimmed" style="width:auto">
84          [size after throttling (empty if not throttled)]
85        </span>
86        [file name (<strike>strikethrough</strike> if file was deleted due to
87            throttling)]
88      </p>
89
90      <button onclick="expandAll();">Expand All</button>
91      <button onclick="collapseAll();">Collapse All</button>
92
93%(summary_tree)s
94
95%(css)s
96%(javascript)s
97
98    </body>
99</html>
100"""
101
102CSS_TEMPLATE = """
103<style>
104  body {
105      font-family: Arial;
106  }
107
108  td.table_header {
109      font-weight: normal;
110  }
111
112  span.size_percent {
113      color: #e8773e;
114      display: inline-block;
115      font-size: 75%%;
116      text-align: right;
117      width: 35px;
118  }
119
120  span.top_size_percent {
121      color: #e8773e;
122      background-color: yellow;
123      display: inline-block;
124      font-size: 75%%;
125      fount-weight: bold;
126      text-align: right;
127      width: 35px;
128  }
129
130  span.size_original {
131      color: sienna;
132      display: inline-block;
133      font-size: 75%%;
134      text-align: right;
135      width: 50px;
136  }
137
138  span.size_trimmed {
139      color: green;
140      display: inline-block;
141      font-size: 75%%;
142      text-align: right;
143      width: %(size_trimmed_width)dpx;
144  }
145
146  ul.tree li {
147      list-style-type: none;
148      position: relative;
149  }
150
151  ul.tree li ul {
152      display: none;
153  }
154
155  ul.tree li.open > ul {
156      display: block;
157  }
158
159  ul.tree li a {
160    color: black;
161    text-decoration: none;
162  }
163
164  ul.tree li a.file {
165    color: blue;
166    text-decoration: underline;
167  }
168
169  ul.tree li a:before {
170      height: 1em;
171      padding:0 .1em;
172      font-size: .8em;
173      display: block;
174      position: absolute;
175      left: -1.3em;
176      top: .2em;
177  }
178
179  ul.tree li > a:not(:last-child):before {
180      content: '+';
181  }
182
183  ul.tree li.open > a:not(:last-child):before {
184      content: '-';
185  }
186</style>
187"""
188
189JAVASCRIPT_TEMPLATE = """
190<script>
191function init() {
192    var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
193    for(var i = 0; i < tree.length; i++){
194        tree[i].addEventListener('click', function(e) {
195            var parent = e.target.parentElement;
196            var classList = parent.classList;
197            if(classList.contains("open")) {
198                classList.remove('open');
199                var opensubs = parent.querySelectorAll(':scope .open');
200                for(var i = 0; i < opensubs.length; i++){
201                    opensubs[i].classList.remove('open');
202                }
203            } else {
204                classList.add('open');
205            }
206        });
207    }
208}
209
210function expandAll() {
211    var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
212    for(var i = 0; i < tree.length; i++){
213        var classList = tree[i].parentElement.classList;
214        if(classList.contains("close")) {
215            classList.remove('close');
216        }
217        classList.add('open');
218    }
219}
220
221function collapseAll() {
222    var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
223    for(var i = 0; i < tree.length; i++){
224        var classList = tree[i].parentElement.classList;
225        if(classList.contains("open")) {
226            classList.remove('open');
227        }
228        classList.add('close');
229    }
230}
231
232// If the current url has `gs_url`, it means the file is opened from Google
233// Storage.
234var gs_url = 'apidata.googleusercontent.com';
235// Base url to open a file from Google Storage
236var gs_file_base_url = '%(gs_file_base_url)s'
237// Path to the result.
238var job_dir = '%(job_dir)s'
239
240function openFile(path) {
241    if(window.location.href.includes(gs_url)) {
242        url = gs_file_base_url + job_dir + '/' + path.substring(3);
243    } else {
244        url = window.location.href + '/' + path;
245    }
246    window.open(url, '_blank');
247}
248</script>
249"""
250
251SIZE_SUMMARY_TEMPLATE = """
252<table>
253  <tr>
254    <td class="table_header">Results collected from test device: </td>
255    <td><span>%(size_client_collected)s</span> </td>
256  </tr>
257  <tr>
258    <td class="table_header">Original size of test results:</td>
259    <td>
260      <span class="size_original" style="font-size:100%%;width:auto">
261        %(size_original)s
262      </span>
263    </td>
264  </tr>
265  <tr>
266    <td class="table_header">Size of test results after throttling:</td>
267    <td>
268      <span class="size_trimmed" style="font-size:100%%;width:auto">
269        %(size_trimmed)s
270      </span>
271    </td>
272  </tr>
273</table>
274"""
275
276SIZE_INFO_TEMPLATE = """
277%(indentation)s<span class="%(size_percent_class)s">%(size_percent)s</span>
278%(indentation)s<span class="size_original">%(size_original)s</span>
279%(indentation)s<span class="size_trimmed">%(size_trimmed)s</span> """
280
281FILE_ENTRY_TEMPLATE = """
282%(indentation)s<li>
283%(indentation)s\t<div>
284%(size_info)s
285%(indentation)s\t\t<a class="file" href="javascript:openFile('%(path)s');" >
286%(indentation)s\t\t\t%(name)s
287%(indentation)s\t\t</a>
288%(indentation)s\t</div>
289%(indentation)s</li>"""
290
291DELETED_FILE_ENTRY_TEMPLATE = """
292%(indentation)s<li>
293%(indentation)s\t<div>
294%(size_info)s
295%(indentation)s\t\t<strike>%(name)s</strike>
296%(indentation)s\t</div>
297%(indentation)s</li>"""
298
299DIR_ENTRY_TEMPLATE = """
300%(indentation)s<li><a>%(size_info)s %(name)s</a>
301%(subdirs)s
302%(indentation)s</li>"""
303
304SUBDIRS_WRAPPER_TEMPLATE = """
305%(indentation)s<ul class="tree">
306%(dirs)s
307%(indentation)s</ul>"""
308
309INDENTATION = '\t'
310
311def _get_size_percent(size_original, total_bytes):
312    """Get the percentage of file size in the parent directory before throttled.
313
314    @param size_original: Original size of the file, in bytes.
315    @param total_bytes: Total size of all files under the parent directory, in
316            bytes.
317    @return: A formatted string of the percentage of file size in the parent
318            directory before throttled.
319    """
320    if total_bytes == 0:
321        return '0%'
322    return '%.1f%%' % (100*float(size_original)/total_bytes)
323
324
325def _get_dirs_html(dirs, parent_path, total_bytes, indentation):
326    """Get the html string for the given directory.
327
328    @param dirs: A list of ResultInfo.
329    @param parent_path: Path to the parent directory.
330    @param total_bytes: Total of the original size of files in the given
331            directories in bytes.
332    @param indentation: Indentation to be used for the html.
333    """
334    if not dirs:
335        return ''
336    summary_html = ''
337    top_size_limit = max([entry.original_size for entry in dirs])
338    # A map between file name to ResultInfo that contains the summary of the
339    # file.
340    entries = dict((list(entry.keys())[0], entry) for entry in dirs)
341    for name in sorted(entries.keys()):
342        entry = entries[name]
343        if not entry.is_dir and re.match(DIR_SUMMARY_PATTERN, name):
344            # Do not include directory summary json files in the html, as they
345            # will be deleted.
346            continue
347
348        size_data = {SIZE_PERCENT: _get_size_percent(entry.original_size,
349                                                     total_bytes),
350                     SIZE_ORIGINAL:
351                        utils_lib.get_size_string(entry.original_size),
352                     SIZE_TRIMMED:
353                        utils_lib.get_size_string(entry.trimmed_size),
354                     INDENTATION_KEY: indentation + 2*INDENTATION}
355        if entry.original_size < top_size_limit:
356            size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_REGULAR
357        else:
358            size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_TOP
359        if entry.trimmed_size == entry.original_size:
360            size_data[SIZE_TRIMMED] = ''
361
362        entry_path = '%s/%s' % (parent_path, name)
363        if not entry.is_dir:
364            # This is a file
365            data = {NAME: name,
366                    PATH: entry_path,
367                    SIZE_INFO: SIZE_INFO_TEMPLATE % size_data,
368                    INDENTATION_KEY: indentation}
369            if entry.original_size > 0 and entry.trimmed_size == 0:
370                summary_html += DELETED_FILE_ENTRY_TEMPLATE % data
371            else:
372                summary_html += FILE_ENTRY_TEMPLATE % data
373        else:
374            subdir_total_size = entry.original_size
375            sub_indentation = indentation + INDENTATION
376            subdirs_html = (
377                    SUBDIRS_WRAPPER_TEMPLATE %
378                    {DIRS: _get_dirs_html(
379                            entry.files, entry_path, subdir_total_size,
380                            sub_indentation),
381                     INDENTATION_KEY: indentation})
382            data = {NAME: entry.name,
383                    SIZE_INFO: SIZE_INFO_TEMPLATE % size_data,
384                    SUBDIRS: subdirs_html,
385                    INDENTATION_KEY: indentation}
386            summary_html += DIR_ENTRY_TEMPLATE % data
387    return summary_html
388
389
390def build(client_collected_bytes, summary, html_file):
391    """Generate an HTML file to visualize the given directory summary.
392
393    @param client_collected_bytes: The total size of results collected from
394            the DUT. The number can be larger than the total file size of the
395            given path, as files can be overwritten or removed.
396    @param summary: A ResultInfo instance containing the directory summary.
397    @param html_file: Path to save the html file to.
398    """
399    size_original = summary.original_size
400    size_trimmed = summary.trimmed_size
401    size_summary_data = {SIZE_CLIENT_COLLECTED:
402                             utils_lib.get_size_string(client_collected_bytes),
403                         SIZE_ORIGINAL:
404                             utils_lib.get_size_string(size_original),
405                         SIZE_TRIMMED:
406                             utils_lib.get_size_string(size_trimmed)}
407    size_trimmed_width = DEFAULT_SIZE_TRIMMED_WIDTH
408    if size_original == size_trimmed:
409        size_summary_data[SIZE_TRIMMED] = NOT_THROTTLED
410        size_trimmed_width = 0
411
412    size_summary = SIZE_SUMMARY_TEMPLATE % size_summary_data
413
414    indentation = INDENTATION
415    dirs_html = _get_dirs_html(
416            summary.files, '..', size_original, indentation + INDENTATION)
417    summary_tree = SUBDIRS_WRAPPER_TEMPLATE % {DIRS: dirs_html,
418                                               INDENTATION_KEY: indentation}
419
420    # job_dir is the path between Autotest `results` folder and the summary html
421    # file, e.g., 123-debug_user/host1. Assume it always contains 2 levels.
422    job_dir_sections = html_file.split(os.sep)[:-1]
423    try:
424        job_dir = '/'.join(job_dir_sections[
425                (job_dir_sections.index('results')+1):])
426    except ValueError:
427        # 'results' is not in the path, default to two levels up of the summary
428        # file.
429        job_dir = '/'.join(job_dir_sections[-2:])
430
431    javascript = (JAVASCRIPT_TEMPLATE %
432                  {GS_FILE_BASE_URL_KEY: GS_FILE_BASE_URL,
433                   JOB_DIR: job_dir})
434    css = CSS_TEMPLATE % {SIZE_TRIMMED_WIDTH: size_trimmed_width}
435    html = PAGE_TEMPLATE % {SIZE_SUMMARY: size_summary,
436                            SUMMARY_TREE: summary_tree,
437                            CSS: css,
438                            JAVASCRIPT: javascript}
439    with open(html_file, 'w') as f:
440        f.write(html)
441