1#!/usr/bin/python3
2#
3# Copyright 2018, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""MLTS benchmark result generator.
18
19Reads a CSV produced by MLTS benchmark and generates
20an HTML page with results summary.
21
22Usage:
23  generate_result [csv input file] [html output file]
24"""
25
26import argparse
27import collections
28import csv
29import os
30import re
31import math
32
33
34class ScoreException(Exception):
35  """Generator base exception type. """
36  pass
37
38
39LatencyResult = collections.namedtuple(
40    'LatencyResult',
41    ['iterations', 'total_time_sec', 'time_freq_start_sec', 'time_freq_step_sec', 'time_freq_sec'])
42
43
44COMPILATION_TYPES = ['compile_without_cache', 'save_to_cache', 'prepare_from_cache']
45BASELINE_COMPILATION_TYPE = COMPILATION_TYPES[0]
46CompilationResult = collections.namedtuple(
47    'CompilationResult',
48    ['cache_size_bytes'] + COMPILATION_TYPES)
49
50
51BenchmarkResult = collections.namedtuple(
52    'BenchmarkResult',
53    ['name', 'backend_type', 'inference_latency', 'max_single_error',
54     'testset_size', 'evaluator_keys', 'evaluator_values', 'validation_errors',
55     'compilation_results'])
56
57
58ResultsWithBaseline = collections.namedtuple(
59    'ResultsWithBaseline',
60    ['baseline', 'other'])
61
62
63BASELINE_BACKEND = 'TFLite_CPU'
64KNOWN_GROUPS = [
65    (re.compile('mobilenet_v1.*quant.*'), 'MobileNet v1 Quantized'),
66    (re.compile('mobilenet_v1.*'), 'MobileNet v1 Float'),
67    (re.compile('mobilenet_v2.*quant.*'), 'MobileNet v2 Quantized'),
68    (re.compile('mobilenet_v2.*'), 'MobileNet v2 Float'),
69    (re.compile('mobilenet_v3.*uint8.*'), 'MobileNet v3 Quantized'),
70    (re.compile('mobilenet_v3.*'), 'MobileNet v3 Float'),
71    (re.compile('tts.*'), 'LSTM Text-to-speech'),
72    (re.compile('asr.*'), 'LSTM Automatic Speech Recognition'),
73]
74
75
76class BenchmarkResultParser:
77  """A helper class to parse the input CSV file."""
78
79  def __init__(self, csvfile):
80    self.csv_reader = csv.reader(filter(lambda row: row[0] != '#', csvfile))
81    self.row = None
82    self.index = 0
83
84  def next(self):
85    """Advance to the next row, returns the current row or None if reaches the end."""
86    try:
87      self.row = next(self.csv_reader)
88    except StopIteration:
89      self.row = None
90    finally:
91      self.index = 0
92      return self.row
93
94  def read_boolean(self):
95    """Read the next CSV cell as a boolean."""
96    s = self.read_typed(str).lower()
97    if s == 'true':
98      return True
99    elif s == 'false':
100      return False
101    else:
102      raise ValueError('Cannot convert \'%s\' to a boolean' % s)
103
104  def read_typed(self, Type):
105    """Read the next CSV cell as the given type."""
106    if Type is bool:
107      return self.read_boolean()
108    entry = self.row[self.index]
109    self.index += 1
110    return Type(entry)
111
112  def read_typed_array(self, Type, length):
113    """Read the next CSV cells as a typed array."""
114    return [self.read_typed(Type) for _ in range(length)]
115
116  def read_latency_result(self):
117    """Read the next CSV cells as a LatencyResult."""
118    result = {}
119    result['iterations'] = self.read_typed(int)
120    result['total_time_sec'] = self.read_typed(float)
121    result['time_freq_start_sec'] = self.read_typed(float)
122    result['time_freq_step_sec'] = self.read_typed(float)
123    time_freq_sec_count = self.read_typed(int)
124    result['time_freq_sec'] = self.read_typed_array(float, time_freq_sec_count)
125    return LatencyResult(**result)
126
127  def read_compilation_result(self):
128    """Read the next CSV cells as a CompilationResult."""
129    result = {}
130    for compilation_type in COMPILATION_TYPES:
131      has_results = self.read_typed(bool)
132      result[compilation_type] = self.read_latency_result() if has_results else None
133    result['cache_size_bytes'] = self.read_typed(int)
134    return CompilationResult(**result)
135
136  def read_benchmark_result(self):
137    """Read the next CSV cells as a BenchmarkResult."""
138    result = {}
139    result['name'] = self.read_typed(str)
140    result['backend_type'] = self.read_typed(str)
141    result['inference_latency'] = self.read_latency_result()
142    result['max_single_error'] = self.read_typed(float)
143    result['testset_size'] = self.read_typed(int)
144    evaluator_keys_count = self.read_typed(int)
145    validation_error_count = self.read_typed(int)
146    result['evaluator_keys'] = self.read_typed_array(str, evaluator_keys_count)
147    result['evaluator_values'] = self.read_typed_array(float, evaluator_keys_count)
148    result['validation_errors'] = self.read_typed_array(str, validation_error_count)
149    result['compilation_results'] = self.read_compilation_result()
150    return BenchmarkResult(**result)
151
152
153def parse_csv_input(input_filename):
154  """Parse input CSV file, returns: (benchmarkInfo, list of BenchmarkResult)."""
155  with open(input_filename, 'r') as csvfile:
156    parser = BenchmarkResultParser(csvfile)
157
158    # First line contain device info
159    benchmark_info = parser.next()
160
161    results = []
162    while parser.next():
163      results.append(parser.read_benchmark_result())
164
165    return (benchmark_info, results)
166
167
168def group_results(results):
169  """Group list of results by their name/backend, returns list of lists."""
170  # Group by name
171  groupings = collections.defaultdict(list)
172  for result in results:
173    groupings[result.name].append(result)
174
175  # Find baseline for each group, make ResultsWithBaseline for each name
176  groupings_baseline = {}
177  for name, results in groupings.items():
178    baseline = next(filter(lambda x: x.backend_type == BASELINE_BACKEND,
179                           results))
180    other = sorted(filter(lambda x: x is not baseline, results),
181                   key=lambda x: x.backend_type)
182    groupings_baseline[name] = ResultsWithBaseline(
183        baseline=baseline,
184        other=other)
185
186  # Merge ResultsWithBaseline for known groups
187  known_groupings_baseline = collections.defaultdict(list)
188  for name, results_with_bl in sorted(groupings_baseline.items()):
189    group_name = name
190    for known_group in KNOWN_GROUPS:
191      if known_group[0].match(results_with_bl.baseline.name):
192        group_name = known_group[1]
193        break
194    known_groupings_baseline[group_name].append(results_with_bl)
195
196  # Turn into a list sorted by name
197  groupings_list = []
198  for name, results_wbl in sorted(known_groupings_baseline.items()):
199    groupings_list.append((name, results_wbl))
200  return groupings_list
201
202
203def get_frequency_graph_min_max(latencies):
204  """Get min and max times of latencies frequency."""
205  mins = []
206  maxs = []
207  for latency in latencies:
208    mins.append(latency.time_freq_start_sec)
209    to_add = len(latency.time_freq_sec) * latency.time_freq_step_sec
210    maxs.append(latency.time_freq_start_sec + to_add)
211  return min(mins), max(maxs)
212
213
214def get_frequency_graph(time_freq_start_sec, time_freq_step_sec, time_freq_sec,
215                        start_sec, end_sec):
216  """Generate input x/y data for latency frequency graph."""
217  left_to_pad = (int((time_freq_start_sec - start_sec) / time_freq_step_sec)
218                 if time_freq_step_sec != 0
219                 else math.inf)
220  end_time = time_freq_start_sec + len(time_freq_sec) * time_freq_step_sec
221  right_to_pad = (int((end_sec - end_time) / time_freq_step_sec)
222                  if time_freq_step_sec != 0
223                  else math.inf)
224
225  # After pading more that 64 values, graphs start to look messy,
226  # bail out in that case.
227  if (left_to_pad + right_to_pad) < 64:
228    left_pad = (['{:.2f}ms'.format(
229        (start_sec + x * time_freq_step_sec) * 1000.0)
230                 for x in range(left_to_pad)], [0] * left_to_pad)
231
232    right_pad = (['{:.2f}ms'.format(
233        (end_time + x * time_freq_step_sec) * 1000.0)
234                  for x in range(right_to_pad)], [0] * right_to_pad)
235  else:
236    left_pad = [[], []]
237    right_pad = [[], []]
238
239  data = (['{:.2f}ms'.format(
240      (time_freq_start_sec + x * time_freq_step_sec) * 1000.0)
241           for x in range(len(time_freq_sec))], time_freq_sec)
242
243  return (left_pad[0] + data[0] + right_pad[0],
244          left_pad[1] + data[1] + right_pad[1])
245
246
247def is_topk_evaluator(evaluator_keys):
248  """Are these evaluator keys from TopK evaluator?"""
249  return (len(evaluator_keys) == 5 and
250          evaluator_keys[0] == 'top_1' and
251          evaluator_keys[1] == 'top_2' and
252          evaluator_keys[2] == 'top_3' and
253          evaluator_keys[3] == 'top_4' and
254          evaluator_keys[4] == 'top_5')
255
256
257def is_melceplogf0_evaluator(evaluator_keys):
258  """Are these evaluator keys from MelCepLogF0 evaluator?"""
259  return (len(evaluator_keys) == 2 and
260          evaluator_keys[0] == 'max_mel_cep_distortion' and
261          evaluator_keys[1] == 'max_log_f0_error')
262
263
264def is_phone_error_rate_evaluator(evaluator_keys):
265  """Are these evaluator keys from PhoneErrorRate evaluator?"""
266  return (len(evaluator_keys) == 1 and
267          evaluator_keys[0] == 'max_phone_error_rate')
268
269
270def generate_accuracy_headers(result):
271  """Accuracy-related headers for result table."""
272  if is_topk_evaluator(result.evaluator_keys):
273    return ACCURACY_HEADERS_TOPK_TEMPLATE
274  elif is_melceplogf0_evaluator(result.evaluator_keys):
275    return ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE
276  elif is_phone_error_rate_evaluator(result.evaluator_keys):
277    return ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE
278  else:
279    return ACCURACY_HEADERS_BASIC_TEMPLATE
280  raise ScoreException('Unknown accuracy headers for: ' + str(result))
281
282
283def get_diff_span(value, same_delta, positive_is_better):
284  if abs(value) < same_delta:
285    return 'same'
286  if positive_is_better and value > 0 or not positive_is_better and value < 0:
287    return 'better'
288  return 'worse'
289
290
291def generate_accuracy_values(baseline, result):
292  """Accuracy-related data for result table."""
293  if is_topk_evaluator(result.evaluator_keys):
294    val = [float(x) * 100.0 for x in result.evaluator_values]
295    if result is baseline:
296      topk = [TOPK_BASELINE_TEMPLATE.format(val=x) for x in val]
297      return ACCURACY_VALUES_TOPK_TEMPLATE.format(
298          top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3],
299          top5=topk[4]
300      )
301    else:
302      base = [float(x) * 100.0 for x in baseline.evaluator_values]
303      diff = [a - b for a, b in zip(val, base)]
304      topk = [TOPK_DIFF_TEMPLATE.format(
305          val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=True))
306              for v, d in zip(val, diff)]
307      return ACCURACY_VALUES_TOPK_TEMPLATE.format(
308          top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3],
309          top5=topk[4]
310      )
311  elif is_melceplogf0_evaluator(result.evaluator_keys):
312    val = [float(x) for x in
313           result.evaluator_values + [result.max_single_error]]
314    if result is baseline:
315      return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format(
316          max_log_f0=MELCEPLOGF0_BASELINE_TEMPLATE.format(
317              val=val[0]),
318          max_mel_cep_distortion=MELCEPLOGF0_BASELINE_TEMPLATE.format(
319              val=val[1]),
320          max_single_error=MELCEPLOGF0_BASELINE_TEMPLATE.format(
321              val=val[2]),
322      )
323    else:
324      base = [float(x) for x in
325              baseline.evaluator_values + [baseline.max_single_error]]
326      diff = [a - b for a, b in zip(val, base)]
327      v = [MELCEPLOGF0_DIFF_TEMPLATE.format(
328          val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False))
329           for v, d in zip(val, diff)]
330      return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format(
331          max_log_f0=v[0],
332          max_mel_cep_distortion=v[1],
333          max_single_error=v[2],
334      )
335  elif is_phone_error_rate_evaluator(result.evaluator_keys):
336    val = [float(x) for x in
337           result.evaluator_values + [result.max_single_error]]
338    if result is baseline:
339      return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format(
340          max_phone_error_rate=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format(
341              val=val[0]),
342          max_single_error=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format(
343              val=val[1]),
344      )
345    else:
346      base = [float(x) for x in
347              baseline.evaluator_values + [baseline.max_single_error]]
348      diff = [a - b for a, b in zip(val, base)]
349      v = [PHONE_ERROR_RATE_DIFF_TEMPLATE.format(
350          val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False))
351           for v, d in zip(val, diff)]
352      return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format(
353          max_phone_error_rate=v[0],
354          max_single_error=v[1],
355      )
356  else:
357    return ACCURACY_VALUES_BASIC_TEMPLATE.format(
358        max_single_error=result.max_single_error,
359    )
360  raise ScoreException('Unknown accuracy values for: ' + str(result))
361
362
363def getchartjs_source():
364  return open(os.path.dirname(os.path.abspath(__file__)) + '/' +
365              CHART_JS_FILE).read()
366
367
368def generate_avg_ms(baseline, latency):
369  """Generate average latency value."""
370  if latency is None:
371    latency = baseline
372
373  result_avg_ms = (latency.total_time_sec / latency.iterations)*1000.0
374  if latency is baseline:
375    return LATENCY_BASELINE_TEMPLATE.format(val=result_avg_ms)
376  baseline_avg_ms = (baseline.total_time_sec / baseline.iterations)*1000.0
377  diff = (result_avg_ms/baseline_avg_ms - 1.0) * 100.0
378  diff_val = result_avg_ms - baseline_avg_ms
379  return LATENCY_DIFF_TEMPLATE.format(
380      val=result_avg_ms,
381      diff=diff,
382      diff_val=diff_val,
383      span=get_diff_span(diff, same_delta=1.0, positive_is_better=False))
384
385
386def generate_result_entry(baseline, result):
387  if result is None:
388    result = baseline
389
390  return RESULT_ENTRY_TEMPLATE.format(
391      row_class='failed' if result.validation_errors else 'normal',
392      name=result.name,
393      backend=result.backend_type,
394      iterations=result.inference_latency.iterations,
395      testset_size=result.testset_size,
396      accuracy_values=generate_accuracy_values(baseline, result),
397      avg_ms=generate_avg_ms(baseline.inference_latency, result.inference_latency))
398
399
400def generate_latency_graph_entry(tag, latency, tmin, tmax):
401  """Generate a single latency graph."""
402  return LATENCY_GRAPH_ENTRY_TEMPLATE.format(
403      tag=tag,
404      i=id(latency),
405      freq_data=get_frequency_graph(latency.time_freq_start_sec,
406                                    latency.time_freq_step_sec,
407                                    latency.time_freq_sec,
408                                    tmin, tmax))
409
410
411def generate_latency_graphs_group(tags, latencies):
412  """Generate a group of latency graphs with the same tmin and tmax."""
413  tmin, tmax = get_frequency_graph_min_max(latencies)
414  return ''.join(
415      generate_latency_graph_entry(tag, latency, tmin, tmax)
416      for tag, latency in zip(tags, latencies))
417
418
419def snake_case_to_title(string):
420  return string.replace('_', ' ').title()
421
422
423def generate_inference_latency_graph_entry(results_with_bl):
424  """Generate a group of latency graphs for inference latencies."""
425  results = [results_with_bl.baseline] + results_with_bl.other
426  tags = [result.backend_type for result in results]
427  latencies = [result.inference_latency for result in results]
428  return generate_latency_graphs_group(tags, latencies)
429
430
431def generate_compilation_latency_graph_entry(results_with_bl):
432  """Generate a group of latency graphs for compilation latencies."""
433  tags = [
434      result.backend_type + ', ' + snake_case_to_title(type)
435      for result in results_with_bl.other
436      for type in COMPILATION_TYPES
437      if getattr(result.compilation_results, type)
438  ]
439  latencies = [
440      getattr(result.compilation_results, type)
441      for result in results_with_bl.other
442      for type in COMPILATION_TYPES
443      if getattr(result.compilation_results, type)
444  ]
445  return generate_latency_graphs_group(tags, latencies)
446
447
448def generate_validation_errors(entries_group):
449  """Generate validation errors table."""
450  errors = []
451  for result_and_bl in entries_group:
452    for result in [result_and_bl.baseline] + result_and_bl.other:
453      for error in result.validation_errors:
454        errors.append((result.name, result.backend_type, error))
455
456  if errors:
457    return VALIDATION_ERRORS_TEMPLATE.format(
458        results=''.join(
459            VALIDATION_ERRORS_ENTRY_TEMPLATE.format(
460                name=name,
461                backend=backend,
462                error=error) for name, backend, error in errors))
463  return ''
464
465
466def generate_compilation_result_entry(result):
467  format_args = {
468      'row_class':
469          'failed' if result.validation_errors else 'normal',
470      'name':
471          result.name,
472      'backend':
473          result.backend_type,
474      'cache_size':
475          f'{result.compilation_results.cache_size_bytes:,}'
476          if result.compilation_results.cache_size_bytes > 0 else '-'
477  }
478  for compilation_type in COMPILATION_TYPES:
479    latency = getattr(result.compilation_results, compilation_type)
480    if latency:
481      format_args[compilation_type + '_iterations'] = f'{latency.iterations}'
482      format_args[compilation_type + '_avg_ms'] = generate_avg_ms(
483          result.compilation_results.compile_without_cache, latency)
484    else:
485      format_args[compilation_type + '_iterations'] = '-'
486      format_args[compilation_type + '_avg_ms'] = '-'
487  return COMPILATION_RESULT_ENTRY_TEMPLATE.format(**format_args)
488
489
490def generate_result(benchmark_info, data):
491  """Turn list of results into HTML."""
492  return MAIN_TEMPLATE.format(
493      jsdeps=getchartjs_source(),
494      device_info=DEVICE_INFO_TEMPLATE.format(
495          benchmark_time=benchmark_info[0],
496          device_info=benchmark_info[1],
497          ),
498      results_list=''.join((
499          RESULT_GROUP_TEMPLATE.format(
500              group_name=entries_name,
501              accuracy_headers=generate_accuracy_headers(
502                  entries_group[0].baseline),
503              results=''.join(
504                  RESULT_ENTRY_WITH_BASELINE_TEMPLATE.format(
505                      baseline=generate_result_entry(
506                          result_and_bl.baseline, None),
507                      other=''.join(
508                          generate_result_entry(
509                              result_and_bl.baseline, x)
510                          for x in result_and_bl.other)
511                  ) for result_and_bl in entries_group),
512              validation_errors=generate_validation_errors(entries_group),
513              latency_graphs=LATENCY_GRAPHS_TEMPLATE.format(
514                  results=''.join(
515                      LATENCY_GRAPH_ENTRY_GROUP_TEMPLATE.format(
516                          name=result_and_bl.baseline.name,
517                          results=generate_inference_latency_graph_entry(result_and_bl)
518                      ) for result_and_bl in entries_group)
519              ),
520              compilation_results=''.join(
521                  COMPILATION_RESULT_ENTRIES_TEMPLATE.format(
522                      entries=''.join(
523                          generate_compilation_result_entry(x) for x in result_and_bl.other)
524                  ) for result_and_bl in entries_group),
525              compilation_latency_graphs=LATENCY_GRAPHS_TEMPLATE.format(
526                  results=''.join(
527                      LATENCY_GRAPH_ENTRY_GROUP_TEMPLATE.format(
528                          name=result_and_bl.baseline.name,
529                          results=generate_compilation_latency_graph_entry(result_and_bl)
530                      ) for result_and_bl in entries_group)
531              ),
532          ) for entries_name, entries_group in group_results(data))
533                          ))
534
535
536def main():
537  parser = argparse.ArgumentParser()
538  parser.add_argument('input', help='input csv filename')
539  parser.add_argument('output', help='output html filename')
540  args = parser.parse_args()
541
542  benchmark_info, data = parse_csv_input(args.input)
543
544  with open(args.output, 'w') as htmlfile:
545    htmlfile.write(generate_result(benchmark_info, data))
546
547
548# -----------------
549# Templates below
550
551MAIN_TEMPLATE = """<!doctype html>
552<html lang='en-US'>
553<head>
554  <meta http-equiv='Content-Type' content='text/html; charset=utf-8'>
555  <script src='https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js'></script>
556  <script>{jsdeps}</script>
557  <title>MLTS results</title>
558  <style>
559    .results {{
560      border-collapse: collapse;
561      width: 100%;
562    }}
563    .results td, .results th {{
564      border: 1px solid #ddd;
565      padding: 6px;
566    }}
567    .results tbody.values {{
568      border-bottom: 8px solid #333;
569    }}
570    span.better {{
571      color: #070;
572    }}
573    span.worse {{
574      color: #700;
575    }}
576    span.same {{
577      color: #000;
578    }}
579    .results tr:nth-child(even) {{background-color: #eee;}}
580    .results tr:hover {{background-color: #ddd;}}
581    .results th {{
582      padding: 10px;
583      font-weight: bold;
584      text-align: left;
585      background-color: #333;
586      color: white;
587    }}
588    .results tr.failed {{
589      background-color: #ffc4ca;
590    }}
591    .group {{
592      padding-top: 25px;
593    }}
594    .group_name {{
595      padding-left: 10px;
596      font-size: 140%;
597      font-weight: bold;
598    }}
599    .section_name {{
600      padding: 10px;
601      font-size: 120%;
602      font-weight: bold;
603    }}
604    .latency_results {{
605       padding: 10px;
606       border: 1px solid #ddd;
607       overflow: hidden;
608    }}
609    .latency_with_baseline {{
610       padding: 10px;
611       border: 1px solid #ddd;
612       overflow: hidden;
613    }}
614  </style>
615</head>
616<body>
617{device_info}
618{results_list}
619</body>
620</html>"""
621
622DEVICE_INFO_TEMPLATE = """<div id='device_info'>
623Benchmark for {device_info}, started at {benchmark_time}
624</div>"""
625
626
627RESULT_GROUP_TEMPLATE = """<div class="group">
628<div class="group_name">{group_name}</div>
629<div class="section_name">Inference results</div>
630<table class="results">
631 <tr>
632   <th>Name</th>
633   <th>Backend</th>
634   <th>Iterations</th>
635   <th>Test set size</th>
636   <th>Average latency ms</th>
637   {accuracy_headers}
638 </tr>
639 {results}
640</table>
641{validation_errors}
642{latency_graphs}
643<div class="section_name">Compilation results</div>
644<table class="results">
645 <tr>
646   <th rowspan="2">Name</th>
647   <th rowspan="2">Backend</th>
648   <th colspan="2">Compile Without Cache</th>
649   <th colspan="2">Save To Cache</th>
650   <th colspan="2">Prepare From Cache</th>
651   <th rowspan="2">Cache size bytes</th>
652 </tr>
653 <tr>
654   <th>Iterations</th>
655   <th>Average latency ms</th>
656   <th>Iterations</th>
657   <th>Average latency ms</th>
658   <th>Iterations</th>
659   <th>Average latency ms</th>
660 </tr>
661 {compilation_results}
662</table>
663{compilation_latency_graphs}
664</div>"""
665
666
667VALIDATION_ERRORS_TEMPLATE = """
668<table class="results">
669 <tr>
670   <th>Name</th>
671   <th>Backend</th>
672   <th>Error</th>
673 </tr>
674 {results}
675</table>"""
676VALIDATION_ERRORS_ENTRY_TEMPLATE = """
677  <tr class="failed">
678    <td>{name}</td>
679    <td>{backend}</td>
680    <td>{error}</td>
681  </tr>
682"""
683
684LATENCY_GRAPHS_TEMPLATE = """
685<div class="latency_results">
686{results}
687</div>
688<div style="clear: left;"></div>
689"""
690
691LATENCY_GRAPH_ENTRY_GROUP_TEMPLATE = """
692<div class="latency_with_baseline" style="float: left;">
693<b>{name}</b>
694{results}
695</div>
696"""
697
698LATENCY_GRAPH_ENTRY_TEMPLATE = """
699<div class="latency_result" style='width: 350px;'>
700{tag}
701<canvas id='latency_chart{i}' class='latency_chart'></canvas>
702  <script>
703   $(function() {{
704       var freqData = {{
705         labels: {freq_data[0]},
706         datasets: [{{
707            data: {freq_data[1]},
708            backgroundColor: 'rgba(255, 99, 132, 0.6)',
709            borderColor:  'rgba(255, 0, 0, 0.6)',
710            borderWidth: 1,
711         }}]
712       }};
713       var ctx = $('#latency_chart{i}')[0].getContext('2d');
714       window.latency_chart{i} = new Chart(ctx,
715        {{
716          type: 'bar',
717          data: freqData,
718          options: {{
719           responsive: true,
720           title: {{
721             display: false,
722             text: 'Latency frequency'
723           }},
724           legend: {{
725             display: false
726           }},
727           scales: {{
728            xAxes: [ {{
729              barPercentage: 1.0,
730              categoryPercentage: 0.9,
731            }}],
732            yAxes: [{{
733              scaleLabel: {{
734                display: false,
735                labelString: 'Iterations Count'
736              }}
737            }}]
738           }}
739         }}
740       }});
741     }});
742    </script>
743</div>
744"""
745
746
747RESULT_ENTRY_WITH_BASELINE_TEMPLATE = """
748 <tbody class="values">
749 {baseline}
750 {other}
751 </tbody>
752"""
753RESULT_ENTRY_TEMPLATE = """
754  <tr class={row_class}>
755   <td>{name}</td>
756   <td>{backend}</td>
757   <td>{iterations:d}</td>
758   <td>{testset_size:d}</td>
759   <td>{avg_ms}</td>
760   {accuracy_values}
761  </tr>"""
762
763COMPILATION_RESULT_ENTRIES_TEMPLATE = """
764 <tbody class="values">
765 {entries}
766 </tbody>
767"""
768COMPILATION_RESULT_ENTRY_TEMPLATE = """
769  <tr class={row_class}>
770   <td>{name}</td>
771   <td>{backend}</td>
772   <td>{compile_without_cache_iterations}</td>
773   <td>{compile_without_cache_avg_ms}</td>
774   <td>{save_to_cache_iterations}</td>
775   <td>{save_to_cache_avg_ms}</td>
776   <td>{prepare_from_cache_iterations}</td>
777   <td>{prepare_from_cache_avg_ms}</td>
778   <td>{cache_size}</td>
779  </tr>"""
780
781LATENCY_BASELINE_TEMPLATE = """{val:.2f}ms"""
782LATENCY_DIFF_TEMPLATE = """{val:.2f}ms <span class='{span}'>
783({diff_val:.2f}ms, {diff:.1f}%)</span>"""
784
785
786ACCURACY_HEADERS_TOPK_TEMPLATE = """
787<th>Top 1</th>
788<th>Top 2</th>
789<th>Top 3</th>
790<th>Top 4</th>
791<th>Top 5</th>
792"""
793ACCURACY_VALUES_TOPK_TEMPLATE = """
794<td>{top1}</td>
795<td>{top2}</td>
796<td>{top3}</td>
797<td>{top4}</td>
798<td>{top5}</td>
799"""
800TOPK_BASELINE_TEMPLATE = """{val:.3f}%"""
801TOPK_DIFF_TEMPLATE = """{val:.3f}% <span class='{span}'>({diff:.1f}%)</span>"""
802
803
804ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE = """
805<th>Max log(F0) error</th>
806<th>Max Mel Cep distortion</th>
807<th>Max scalar error</th>
808"""
809
810ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE = """
811<td>{max_log_f0}</td>
812<td>{max_mel_cep_distortion}</td>
813<td>{max_single_error}</td>
814"""
815
816MELCEPLOGF0_BASELINE_TEMPLATE = """{val:.2E}"""
817MELCEPLOGF0_DIFF_TEMPLATE = \
818"""{val:.2E} <span class='{span}'>({diff:.1f}%)</span>"""
819
820
821ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE = """
822<th>Max phone error rate</th>
823<th>Max scalar error</th>
824"""
825
826ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE = """
827<td>{max_phone_error_rate}</td>
828<td>{max_single_error}</td>
829"""
830
831PHONE_ERROR_RATE_BASELINE_TEMPLATE = """{val:.3f}"""
832PHONE_ERROR_RATE_DIFF_TEMPLATE = \
833"""{val:.3f} <span class='{span}'>({diff:.1f}%)</span>"""
834
835
836ACCURACY_HEADERS_BASIC_TEMPLATE = """
837<th>Max single scalar error</th>
838"""
839
840
841ACCURACY_VALUES_BASIC_TEMPLATE = """
842<td>{max_single_error:.2f}</td>
843"""
844
845CHART_JS_FILE = 'Chart.bundle.min.js'
846
847if __name__ == '__main__':
848  main()
849