1# Lint as: python2, python3 2# Copyright 2017 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" 7This is a utility to build an html page based on the directory summaries 8collected during the test. 9""" 10 11import os 12import re 13 14import common 15from autotest_lib.client.bin.result_tools import utils_lib 16from autotest_lib.client.common_lib import global_config 17 18 19CONFIG = global_config.global_config 20# Base url to open a file from Google Storage 21GS_FILE_BASE_URL = CONFIG.get_config_value('CROS', 'gs_file_base_url') 22 23# Default width of `size_trimmed_width`. If throttle is not applied, the block 24# of `size_trimmed_width` will be set to minimum to make the view more compact. 25DEFAULT_SIZE_TRIMMED_WIDTH = 50 26 27DEFAULT_RESULT_SUMMARY_NAME = 'result_summary.html' 28 29DIR_SUMMARY_PATTERN = 'dir_summary_\d+.json' 30 31# ================================================== 32# Following are key names used in the html templates: 33 34CSS = 'css' 35DIRS = 'dirs' 36GS_FILE_BASE_URL_KEY = 'gs_file_base_url' 37INDENTATION_KEY = 'indentation' 38JAVASCRIPT = 'javascript' 39JOB_DIR = 'job_dir' 40NAME = 'name' 41PATH = 'path' 42 43SIZE_CLIENT_COLLECTED = 'size_client_collected' 44 45SIZE_INFO = 'size_info' 46SIZE_ORIGINAL = 'size_original' 47SIZE_PERCENT = 'size_percent' 48SIZE_PERCENT_CLASS = 'size_percent_class' 49SIZE_PERCENT_CLASS_REGULAR = 'size_percent' 50SIZE_PERCENT_CLASS_TOP = 'top_size_percent' 51SIZE_SUMMARY = 'size_summary' 52SIZE_TRIMMED = 'size_trimmed' 53 54# Width of `size_trimmed` block` 55SIZE_TRIMMED_WIDTH = 'size_trimmed_width' 56 57SUBDIRS = 'subdirs' 58SUMMARY_TREE = 'summary_tree' 59# ================================================== 60 61# Text to show when test result is not throttled. 62NOT_THROTTLED = '(Not throttled)' 63 64 65PAGE_TEMPLATE = """ 66<!DOCTYPE html> 67 <html> 68 <body onload="init()"> 69 <h3>Summary of test results</h3> 70%(size_summary)s 71 <p> 72 <b> 73 Display format of a file or directory: 74 </b> 75 </p> 76 <p> 77 <span class="size_percent" style="width:auto"> 78 [percentage of size in the parent directory] 79 </span> 80 <span class="size_original" style="width:auto"> 81 [original size] 82 </span> 83 <span class="size_trimmed" style="width:auto"> 84 [size after throttling (empty if not throttled)] 85 </span> 86 [file name (<strike>strikethrough</strike> if file was deleted due to 87 throttling)] 88 </p> 89 90 <button onclick="expandAll();">Expand All</button> 91 <button onclick="collapseAll();">Collapse All</button> 92 93%(summary_tree)s 94 95%(css)s 96%(javascript)s 97 98 </body> 99</html> 100""" 101 102CSS_TEMPLATE = """ 103<style> 104 body { 105 font-family: Arial; 106 } 107 108 td.table_header { 109 font-weight: normal; 110 } 111 112 span.size_percent { 113 color: #e8773e; 114 display: inline-block; 115 font-size: 75%%; 116 text-align: right; 117 width: 35px; 118 } 119 120 span.top_size_percent { 121 color: #e8773e; 122 background-color: yellow; 123 display: inline-block; 124 font-size: 75%%; 125 fount-weight: bold; 126 text-align: right; 127 width: 35px; 128 } 129 130 span.size_original { 131 color: sienna; 132 display: inline-block; 133 font-size: 75%%; 134 text-align: right; 135 width: 50px; 136 } 137 138 span.size_trimmed { 139 color: green; 140 display: inline-block; 141 font-size: 75%%; 142 text-align: right; 143 width: %(size_trimmed_width)dpx; 144 } 145 146 ul.tree li { 147 list-style-type: none; 148 position: relative; 149 } 150 151 ul.tree li ul { 152 display: none; 153 } 154 155 ul.tree li.open > ul { 156 display: block; 157 } 158 159 ul.tree li a { 160 color: black; 161 text-decoration: none; 162 } 163 164 ul.tree li a.file { 165 color: blue; 166 text-decoration: underline; 167 } 168 169 ul.tree li a:before { 170 height: 1em; 171 padding:0 .1em; 172 font-size: .8em; 173 display: block; 174 position: absolute; 175 left: -1.3em; 176 top: .2em; 177 } 178 179 ul.tree li > a:not(:last-child):before { 180 content: '+'; 181 } 182 183 ul.tree li.open > a:not(:last-child):before { 184 content: '-'; 185 } 186</style> 187""" 188 189JAVASCRIPT_TEMPLATE = """ 190<script> 191function init() { 192 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 193 for(var i = 0; i < tree.length; i++){ 194 tree[i].addEventListener('click', function(e) { 195 var parent = e.target.parentElement; 196 var classList = parent.classList; 197 if(classList.contains("open")) { 198 classList.remove('open'); 199 var opensubs = parent.querySelectorAll(':scope .open'); 200 for(var i = 0; i < opensubs.length; i++){ 201 opensubs[i].classList.remove('open'); 202 } 203 } else { 204 classList.add('open'); 205 } 206 }); 207 } 208} 209 210function expandAll() { 211 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 212 for(var i = 0; i < tree.length; i++){ 213 var classList = tree[i].parentElement.classList; 214 if(classList.contains("close")) { 215 classList.remove('close'); 216 } 217 classList.add('open'); 218 } 219} 220 221function collapseAll() { 222 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 223 for(var i = 0; i < tree.length; i++){ 224 var classList = tree[i].parentElement.classList; 225 if(classList.contains("open")) { 226 classList.remove('open'); 227 } 228 classList.add('close'); 229 } 230} 231 232// If the current url has `gs_url`, it means the file is opened from Google 233// Storage. 234var gs_url = 'apidata.googleusercontent.com'; 235// Base url to open a file from Google Storage 236var gs_file_base_url = '%(gs_file_base_url)s' 237// Path to the result. 238var job_dir = '%(job_dir)s' 239 240function openFile(path) { 241 if(window.location.href.includes(gs_url)) { 242 url = gs_file_base_url + job_dir + '/' + path.substring(3); 243 } else { 244 url = window.location.href + '/' + path; 245 } 246 window.open(url, '_blank'); 247} 248</script> 249""" 250 251SIZE_SUMMARY_TEMPLATE = """ 252<table> 253 <tr> 254 <td class="table_header">Results collected from test device: </td> 255 <td><span>%(size_client_collected)s</span> </td> 256 </tr> 257 <tr> 258 <td class="table_header">Original size of test results:</td> 259 <td> 260 <span class="size_original" style="font-size:100%%;width:auto"> 261 %(size_original)s 262 </span> 263 </td> 264 </tr> 265 <tr> 266 <td class="table_header">Size of test results after throttling:</td> 267 <td> 268 <span class="size_trimmed" style="font-size:100%%;width:auto"> 269 %(size_trimmed)s 270 </span> 271 </td> 272 </tr> 273</table> 274""" 275 276SIZE_INFO_TEMPLATE = """ 277%(indentation)s<span class="%(size_percent_class)s">%(size_percent)s</span> 278%(indentation)s<span class="size_original">%(size_original)s</span> 279%(indentation)s<span class="size_trimmed">%(size_trimmed)s</span> """ 280 281FILE_ENTRY_TEMPLATE = """ 282%(indentation)s<li> 283%(indentation)s\t<div> 284%(size_info)s 285%(indentation)s\t\t<a class="file" href="javascript:openFile('%(path)s');" > 286%(indentation)s\t\t\t%(name)s 287%(indentation)s\t\t</a> 288%(indentation)s\t</div> 289%(indentation)s</li>""" 290 291DELETED_FILE_ENTRY_TEMPLATE = """ 292%(indentation)s<li> 293%(indentation)s\t<div> 294%(size_info)s 295%(indentation)s\t\t<strike>%(name)s</strike> 296%(indentation)s\t</div> 297%(indentation)s</li>""" 298 299DIR_ENTRY_TEMPLATE = """ 300%(indentation)s<li><a>%(size_info)s %(name)s</a> 301%(subdirs)s 302%(indentation)s</li>""" 303 304SUBDIRS_WRAPPER_TEMPLATE = """ 305%(indentation)s<ul class="tree"> 306%(dirs)s 307%(indentation)s</ul>""" 308 309INDENTATION = '\t' 310 311def _get_size_percent(size_original, total_bytes): 312 """Get the percentage of file size in the parent directory before throttled. 313 314 @param size_original: Original size of the file, in bytes. 315 @param total_bytes: Total size of all files under the parent directory, in 316 bytes. 317 @return: A formatted string of the percentage of file size in the parent 318 directory before throttled. 319 """ 320 if total_bytes == 0: 321 return '0%' 322 return '%.1f%%' % (100*float(size_original)/total_bytes) 323 324 325def _get_dirs_html(dirs, parent_path, total_bytes, indentation): 326 """Get the html string for the given directory. 327 328 @param dirs: A list of ResultInfo. 329 @param parent_path: Path to the parent directory. 330 @param total_bytes: Total of the original size of files in the given 331 directories in bytes. 332 @param indentation: Indentation to be used for the html. 333 """ 334 if not dirs: 335 return '' 336 summary_html = '' 337 top_size_limit = max([entry.original_size for entry in dirs]) 338 # A map between file name to ResultInfo that contains the summary of the 339 # file. 340 entries = dict((list(entry.keys())[0], entry) for entry in dirs) 341 for name in sorted(entries.keys()): 342 entry = entries[name] 343 if not entry.is_dir and re.match(DIR_SUMMARY_PATTERN, name): 344 # Do not include directory summary json files in the html, as they 345 # will be deleted. 346 continue 347 348 size_data = {SIZE_PERCENT: _get_size_percent(entry.original_size, 349 total_bytes), 350 SIZE_ORIGINAL: 351 utils_lib.get_size_string(entry.original_size), 352 SIZE_TRIMMED: 353 utils_lib.get_size_string(entry.trimmed_size), 354 INDENTATION_KEY: indentation + 2*INDENTATION} 355 if entry.original_size < top_size_limit: 356 size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_REGULAR 357 else: 358 size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_TOP 359 if entry.trimmed_size == entry.original_size: 360 size_data[SIZE_TRIMMED] = '' 361 362 entry_path = '%s/%s' % (parent_path, name) 363 if not entry.is_dir: 364 # This is a file 365 data = {NAME: name, 366 PATH: entry_path, 367 SIZE_INFO: SIZE_INFO_TEMPLATE % size_data, 368 INDENTATION_KEY: indentation} 369 if entry.original_size > 0 and entry.trimmed_size == 0: 370 summary_html += DELETED_FILE_ENTRY_TEMPLATE % data 371 else: 372 summary_html += FILE_ENTRY_TEMPLATE % data 373 else: 374 subdir_total_size = entry.original_size 375 sub_indentation = indentation + INDENTATION 376 subdirs_html = ( 377 SUBDIRS_WRAPPER_TEMPLATE % 378 {DIRS: _get_dirs_html( 379 entry.files, entry_path, subdir_total_size, 380 sub_indentation), 381 INDENTATION_KEY: indentation}) 382 data = {NAME: entry.name, 383 SIZE_INFO: SIZE_INFO_TEMPLATE % size_data, 384 SUBDIRS: subdirs_html, 385 INDENTATION_KEY: indentation} 386 summary_html += DIR_ENTRY_TEMPLATE % data 387 return summary_html 388 389 390def build(client_collected_bytes, summary, html_file): 391 """Generate an HTML file to visualize the given directory summary. 392 393 @param client_collected_bytes: The total size of results collected from 394 the DUT. The number can be larger than the total file size of the 395 given path, as files can be overwritten or removed. 396 @param summary: A ResultInfo instance containing the directory summary. 397 @param html_file: Path to save the html file to. 398 """ 399 size_original = summary.original_size 400 size_trimmed = summary.trimmed_size 401 size_summary_data = {SIZE_CLIENT_COLLECTED: 402 utils_lib.get_size_string(client_collected_bytes), 403 SIZE_ORIGINAL: 404 utils_lib.get_size_string(size_original), 405 SIZE_TRIMMED: 406 utils_lib.get_size_string(size_trimmed)} 407 size_trimmed_width = DEFAULT_SIZE_TRIMMED_WIDTH 408 if size_original == size_trimmed: 409 size_summary_data[SIZE_TRIMMED] = NOT_THROTTLED 410 size_trimmed_width = 0 411 412 size_summary = SIZE_SUMMARY_TEMPLATE % size_summary_data 413 414 indentation = INDENTATION 415 dirs_html = _get_dirs_html( 416 summary.files, '..', size_original, indentation + INDENTATION) 417 summary_tree = SUBDIRS_WRAPPER_TEMPLATE % {DIRS: dirs_html, 418 INDENTATION_KEY: indentation} 419 420 # job_dir is the path between Autotest `results` folder and the summary html 421 # file, e.g., 123-debug_user/host1. Assume it always contains 2 levels. 422 job_dir_sections = html_file.split(os.sep)[:-1] 423 try: 424 job_dir = '/'.join(job_dir_sections[ 425 (job_dir_sections.index('results')+1):]) 426 except ValueError: 427 # 'results' is not in the path, default to two levels up of the summary 428 # file. 429 job_dir = '/'.join(job_dir_sections[-2:]) 430 431 javascript = (JAVASCRIPT_TEMPLATE % 432 {GS_FILE_BASE_URL_KEY: GS_FILE_BASE_URL, 433 JOB_DIR: job_dir}) 434 css = CSS_TEMPLATE % {SIZE_TRIMMED_WIDTH: size_trimmed_width} 435 html = PAGE_TEMPLATE % {SIZE_SUMMARY: size_summary, 436 SUMMARY_TREE: summary_tree, 437 CSS: css, 438 JAVASCRIPT: javascript} 439 with open(html_file, 'w') as f: 440 f.write(html) 441