1// Copyright 2020 The Marl Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// https://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// benchdiff is a tool that compares two Google benchmark results and displays 16// sorted performance differences. 17package main 18 19import ( 20 "errors" 21 "flag" 22 "fmt" 23 "io/ioutil" 24 "os" 25 "path/filepath" 26 "sort" 27 "text/tabwriter" 28 "time" 29 30 "../../bench" 31) 32 33var ( 34 minDiff = flag.Duration("min-diff", time.Microsecond*10, "Filter away time diffs less than this duration") 35 minRelDiff = flag.Float64("min-rel-diff", 0.01, "Filter away absolute relative diffs between [1, 1+x]") 36) 37 38func main() { 39 flag.ErrHelp = errors.New("benchdiff is a tool to compare two benchmark results") 40 flag.Parse() 41 flag.Usage = func() { 42 fmt.Fprintln(os.Stderr, "benchdiff <benchmark-a> <benchmark-b>") 43 flag.PrintDefaults() 44 } 45 46 args := flag.Args() 47 if len(args) < 2 { 48 flag.Usage() 49 os.Exit(1) 50 } 51 52 pathA, pathB := args[0], args[1] 53 54 if err := run(pathA, pathB); err != nil { 55 fmt.Fprintln(os.Stderr, err) 56 os.Exit(-1) 57 } 58} 59 60func run(pathA, pathB string) error { 61 fileA, err := ioutil.ReadFile(pathA) 62 if err != nil { 63 return err 64 } 65 benchA, err := bench.Parse(string(fileA)) 66 if err != nil { 67 return err 68 } 69 70 fileB, err := ioutil.ReadFile(pathB) 71 if err != nil { 72 return err 73 } 74 benchB, err := bench.Parse(string(fileB)) 75 if err != nil { 76 return err 77 } 78 79 compare(benchA, benchB, fileName(pathA), fileName(pathB)) 80 81 return nil 82} 83 84func fileName(path string) string { 85 _, name := filepath.Split(path) 86 return name 87} 88 89func compare(benchA, benchB bench.Benchmark, nameA, nameB string) { 90 type times struct { 91 a time.Duration 92 b time.Duration 93 } 94 byName := map[string]times{} 95 for _, test := range benchA.Tests { 96 byName[test.Name] = times{a: test.Duration} 97 } 98 for _, test := range benchB.Tests { 99 t := byName[test.Name] 100 t.b = test.Duration 101 byName[test.Name] = t 102 } 103 104 type delta struct { 105 name string 106 times times 107 relDiff float64 108 absRelDiff float64 109 } 110 deltas := []delta{} 111 for name, times := range byName { 112 if times.a == 0 || times.b == 0 { 113 continue // Assuming test was missing from a or b 114 } 115 diff := times.b - times.a 116 absDiff := diff 117 if absDiff < 0 { 118 absDiff = -absDiff 119 } 120 if absDiff < *minDiff { 121 continue 122 } 123 124 relDiff := float64(times.b) / float64(times.a) 125 absRelDiff := relDiff 126 if absRelDiff < 1 { 127 absRelDiff = 1.0 / absRelDiff 128 } 129 if absRelDiff < (1.0 + *minRelDiff) { 130 continue 131 } 132 133 d := delta{ 134 name: name, 135 times: times, 136 relDiff: relDiff, 137 absRelDiff: absRelDiff, 138 } 139 deltas = append(deltas, d) 140 } 141 142 sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff }) 143 144 w := tabwriter.NewWriter(os.Stdout, 1, 1, 0, ' ', 0) 145 fmt.Fprintf(w, "Delta\t | Test name\t | (A) %v\t | (B) %v\n", nameA, nameB) 146 for _, delta := range deltas { 147 sign, diff := "+", delta.times.b-delta.times.a 148 if diff < 0 { 149 sign, diff = "-", -diff 150 } 151 fmt.Fprintf(w, "%v%.2fx %v%+v\t | %v\t | %v\t | %v\n", sign, delta.absRelDiff, sign, diff, delta.name, delta.times.a, delta.times.b) 152 } 153 w.Flush() 154} 155