1#!/usr/bin/env python3
2
3import subprocess, os, sys
4import xml.etree.ElementTree as ET
5from collections import defaultdict
6from statistics import median, stdev
7from datetime import datetime
8
9def get_commit_hash():
10    res = subprocess.run('git rev-parse HEAD'.split(), check=True, stdout=subprocess.PIPE, universal_newlines=True)
11    return res.stdout.strip()
12
13if len(sys.argv) < 2:
14    print('Usage: {} benchmark-binary'.format(sys.argv[0]))
15    exit(1)
16
17
18num_runs = 10
19data = defaultdict(list)
20
21
22def parse_file(file):
23
24    def recursive_search(node):
25        if node.tag == 'TestCase':
26            results = node.find('OverallResult')
27            time = results.get('durationInSeconds')
28            data[node.get('name')].append(float(time))
29        elif node.tag in ('Group', 'Catch'):
30            for child in node:
31                recursive_search(child)
32
33    tree = ET.parse(file)
34    recursive_search(tree.getroot())
35
36def run_benchmarks(binary):
37    call = [binary] + '-d yes -r xml -o'.split()
38    for i in range(num_runs):
39        file = 'temp{}.xml'.format(i)
40        print('Run number {}'.format(i))
41        subprocess.run(call + [file])
42        parse_file(file)
43        # Remove file right after parsing, because benchmark output can be big
44        os.remove(file)
45
46
47# Run benchmarks
48run_benchmarks(sys.argv[1])
49
50result_file = '{:%Y-%m-%dT%H-%M-%S}-{}.result'.format(datetime.now(), get_commit_hash())
51
52
53print('Writing results to {}'.format(result_file))
54with open(result_file, 'w') as file:
55    for k in sorted(data):
56        file.write('{}: median: {} (s), stddev: {} (s)\n'.format(k, median(data[k]), stdev(data[k])))
57