1// Copyright 2020 The Marl Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// benchdiff is a tool that compares two Google benchmark results and displays
16// sorted performance differences.
17package main
18
19import (
20	"errors"
21	"flag"
22	"fmt"
23	"io/ioutil"
24	"os"
25	"path/filepath"
26	"sort"
27	"text/tabwriter"
28	"time"
29
30	"../../bench"
31)
32
33var (
34	minDiff    = flag.Duration("min-diff", time.Microsecond*10, "Filter away time diffs less than this duration")
35	minRelDiff = flag.Float64("min-rel-diff", 0.01, "Filter away absolute relative diffs between [1, 1+x]")
36)
37
38func main() {
39	flag.ErrHelp = errors.New("benchdiff is a tool to compare two benchmark results")
40	flag.Parse()
41	flag.Usage = func() {
42		fmt.Fprintln(os.Stderr, "benchdiff <benchmark-a> <benchmark-b>")
43		flag.PrintDefaults()
44	}
45
46	args := flag.Args()
47	if len(args) < 2 {
48		flag.Usage()
49		os.Exit(1)
50	}
51
52	pathA, pathB := args[0], args[1]
53
54	if err := run(pathA, pathB); err != nil {
55		fmt.Fprintln(os.Stderr, err)
56		os.Exit(-1)
57	}
58}
59
60func run(pathA, pathB string) error {
61	fileA, err := ioutil.ReadFile(pathA)
62	if err != nil {
63		return err
64	}
65	benchA, err := bench.Parse(string(fileA))
66	if err != nil {
67		return err
68	}
69
70	fileB, err := ioutil.ReadFile(pathB)
71	if err != nil {
72		return err
73	}
74	benchB, err := bench.Parse(string(fileB))
75	if err != nil {
76		return err
77	}
78
79	compare(benchA, benchB, fileName(pathA), fileName(pathB))
80
81	return nil
82}
83
84func fileName(path string) string {
85	_, name := filepath.Split(path)
86	return name
87}
88
89func compare(benchA, benchB bench.Benchmark, nameA, nameB string) {
90	type times struct {
91		a time.Duration
92		b time.Duration
93	}
94	byName := map[string]times{}
95	for _, test := range benchA.Tests {
96		byName[test.Name] = times{a: test.Duration}
97	}
98	for _, test := range benchB.Tests {
99		t := byName[test.Name]
100		t.b = test.Duration
101		byName[test.Name] = t
102	}
103
104	type delta struct {
105		name       string
106		times      times
107		relDiff    float64
108		absRelDiff float64
109	}
110	deltas := []delta{}
111	for name, times := range byName {
112		if times.a == 0 || times.b == 0 {
113			continue // Assuming test was missing from a or b
114		}
115		diff := times.b - times.a
116		absDiff := diff
117		if absDiff < 0 {
118			absDiff = -absDiff
119		}
120		if absDiff < *minDiff {
121			continue
122		}
123
124		relDiff := float64(times.b) / float64(times.a)
125		absRelDiff := relDiff
126		if absRelDiff < 1 {
127			absRelDiff = 1.0 / absRelDiff
128		}
129		if absRelDiff < (1.0 + *minRelDiff) {
130			continue
131		}
132
133		d := delta{
134			name:       name,
135			times:      times,
136			relDiff:    relDiff,
137			absRelDiff: absRelDiff,
138		}
139		deltas = append(deltas, d)
140	}
141
142	sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
143
144	w := tabwriter.NewWriter(os.Stdout, 1, 1, 0, ' ', 0)
145	fmt.Fprintf(w, "Delta\t | Test name\t | (A) %v\t | (B) %v\n", nameA, nameB)
146	for _, delta := range deltas {
147		sign, diff := "+", delta.times.b-delta.times.a
148		if diff < 0 {
149			sign, diff = "-", -diff
150		}
151		fmt.Fprintf(w, "%v%.2fx %v%+v\t | %v\t | %v\t | %v\n", sign, delta.absRelDiff, sign, diff, delta.name, delta.times.a, delta.times.b)
152	}
153	w.Flush()
154}
155