1// Copyright 2017 syzkaller project authors. All rights reserved.
2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
3
4// syz-benchcmp visualizes syz-manager benchmarking results.
5// First, run syz-manager with -bench=old flag.
6// Then, do experimental modifications and run syz-manager again with -bench=new flag.
7// Then, run syz-benchcmp old new.
8package main
9
10import (
11	"bufio"
12	"encoding/json"
13	"flag"
14	"fmt"
15	"html/template"
16	"io/ioutil"
17	"os"
18	"os/exec"
19	"path/filepath"
20	"sort"
21)
22
23var (
24	flagSkip = flag.Int("skip", -30, "skip that many seconds after start (skip first 20% by default)")
25)
26
27type Graph struct {
28	Name    string
29	Headers []string
30	Points  []Point
31}
32
33type Point struct {
34	Time uint64
35	Vals []uint64
36}
37
38func main() {
39	flag.Parse()
40	if len(flag.Args()) == 0 {
41		fmt.Fprintf(os.Stderr, "usage: syz-benchcmp [flags] bench_file0 [bench_file1 [bench_file2]]...\n")
42		flag.PrintDefaults()
43		os.Exit(1)
44	}
45
46	graphs := []*Graph{
47		{Name: "coverage"},
48		{Name: "corpus"},
49		{Name: "exec total"},
50		{Name: "crash types"},
51	}
52	for i, fname := range flag.Args() {
53		data := readFile(fname)
54		addExecSpeed(data)
55		for _, g := range graphs {
56			g.Headers = append(g.Headers, filepath.Base(fname))
57			for _, v := range data {
58				pt := Point{
59					Time: v["fuzzing"],
60					Vals: make([]uint64, len(flag.Args())),
61				}
62				pt.Vals[i] = v[g.Name]
63				g.Points = append(g.Points, pt)
64			}
65		}
66	}
67	for _, g := range graphs {
68		if len(g.Points) == 0 {
69			failf("no data points")
70		}
71		sort.Sort(pointSlice(g.Points))
72		skipStart(g)
73		restoreMissingPoints(g)
74	}
75	printFinalStats(graphs)
76	display(graphs)
77}
78
79func readFile(fname string) (data []map[string]uint64) {
80	f, err := os.Open(fname)
81	if err != nil {
82		failf("failed to open input file: %v", err)
83	}
84	defer f.Close()
85	dec := json.NewDecoder(bufio.NewReader(f))
86	for dec.More() {
87		v := make(map[string]uint64)
88		if err := dec.Decode(&v); err != nil {
89			failf("failed to decode input file %v: %v", fname, err)
90		}
91		data = append(data, v)
92	}
93	return
94}
95
96func addExecSpeed(data []map[string]uint64) {
97	// Speed between consecutive samples is very unstable.
98	const (
99		window = 100
100		step   = 10
101	)
102	for i := window; i < len(data); i += step {
103		cur := data[i]
104		prev := data[i-window]
105		dx := cur["exec total"] - prev["exec total"]
106		dt := cur["fuzzing"] - prev["fuzzing"]
107		cur["exec speed"] = dx * 1000 / dt
108	}
109}
110
111func skipStart(g *Graph) {
112	skipTime := uint64(*flagSkip)
113	if *flagSkip < 0 {
114		// Negative skip means percents.
115		max := g.Points[len(g.Points)-1].Time
116		skipTime = max * -skipTime / 100
117	}
118	if skipTime > 0 {
119		skip := sort.Search(len(g.Points), func(i int) bool {
120			return g.Points[i].Time > skipTime
121		})
122		g.Points = g.Points[skip:]
123	}
124}
125
126func restoreMissingPoints(g *Graph) {
127	for i := range g.Headers {
128		// Find previous and next non-zero point for each zero point,
129		// and restore its value with linear inerpolation.
130		type Pt struct {
131			Time uint64
132			Val  uint64
133		}
134		var prev Pt
135		prevs := make(map[uint64]Pt)
136		for _, pt := range g.Points {
137			if pt.Vals[i] != 0 {
138				prev = Pt{pt.Time, pt.Vals[i]}
139				continue
140			}
141			prevs[pt.Time] = prev
142		}
143		var next Pt
144		for pti := len(g.Points) - 1; pti >= 0; pti-- {
145			pt := g.Points[pti]
146			if pt.Vals[i] != 0 {
147				next = Pt{pt.Time, pt.Vals[i]}
148				continue
149			}
150			prev := prevs[pt.Time]
151			if prev.Val == 0 || next.Val == 0 {
152				continue
153			}
154			pt.Vals[i] = prev.Val
155			if next.Time != prev.Time {
156				// Use signed calculations as corpus can go backwards.
157				pt.Vals[i] += uint64(int64(next.Val-prev.Val) * int64(pt.Time-prev.Time) / int64(next.Time-prev.Time))
158			}
159		}
160	}
161}
162
163func printFinalStats(graphs []*Graph) {
164	for i := 1; i < len(graphs[0].Headers); i++ {
165		fmt.Printf("%-12v%16v%16v%16v\n", "", graphs[0].Headers[0], graphs[0].Headers[i], "diff")
166		for _, g := range graphs {
167			lastNonZero := func(x int) uint64 {
168				for j := len(g.Points) - 1; j >= 0; j-- {
169					if v := g.Points[j].Vals[x]; v != 0 {
170						return v
171					}
172				}
173				return 0
174			}
175			old := lastNonZero(0)
176			new := lastNonZero(i)
177			fmt.Printf("%-12v%16v%16v%+16d\n", g.Name, old, new, int64(new-old))
178		}
179		fmt.Printf("\n")
180	}
181}
182
183func display(graphs []*Graph) {
184	outf, err := ioutil.TempFile("", "")
185	if err != nil {
186		failf("failed to create temp file: %v", err)
187	}
188	if err := htmlTemplate.Execute(outf, graphs); err != nil {
189		failf("failed to execute template: %v", err)
190	}
191	outf.Close()
192	name := outf.Name() + ".html"
193	if err := os.Rename(outf.Name(), name); err != nil {
194		failf("failed to rename file: %v", err)
195	}
196	if err := exec.Command("xdg-open", name).Start(); err != nil {
197		failf("failed to start browser: %v", err)
198	}
199}
200
201type pointSlice []Point
202
203func (a pointSlice) Len() int           { return len(a) }
204func (a pointSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
205func (a pointSlice) Less(i, j int) bool { return a[i].Time < a[j].Time }
206
207func failf(msg string, args ...interface{}) {
208	fmt.Fprintf(os.Stderr, msg+"\n", args...)
209	os.Exit(1)
210}
211
212var htmlTemplate = template.Must(
213	template.New("").Parse(`
214<!doctype html>
215<html>
216  <head>
217    <title>Syzkaller Bench</title>
218    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
219    <script type="text/javascript">
220      google.load("visualization", "1", {packages:["corechart"]});
221      google.setOnLoadCallback(drawCharts);
222      function drawCharts() {
223        {{range $id, $graph := .}}
224        {
225          var data = new google.visualization.DataTable();
226          data.addColumn({type: 'number'});
227          {{range $graph.Headers}}
228            data.addColumn({type: 'number', label: '{{.}}'});
229          {{end}}
230          data.addRows([
231            {{range $graph.Points}} [ {{.Time}}, {{range .Vals}} {{if .}} {{.}} {{end}}, {{end}}
232          ],
233          {{end}}
234          ]);
235          new google.visualization.LineChart(document.getElementById('graph_div_{{$id}}')).
236            draw(data, {
237              title: '{{$graph.Name}}',
238              width: "100%",
239              height: document.documentElement.clientHeight * 0.48,
240              legend: {position: "in"},
241              focusTarget: "category",
242              hAxis: {title: "Time, sec"},
243              chartArea: {left: "5%", top: "5%", width: "90%", height:"85%"}
244            })
245        }
246        {{end}}
247      }
248    </script>
249</head>
250<body>
251  <table style="width: 100%; height: 98vh">
252    <tr>
253      <td style="width: 50%"> <div id="graph_div_0"></div> </td>
254      <td style="width: 50%"> <div id="graph_div_2"></div> </td>
255    </tr>
256    <tr>
257      <td style="width: 50%"> <div id="graph_div_1"></div> </td>
258      <td style="width: 50%"> <div id="graph_div_3"></div> </td>
259    </tr>
260  </table>
261</body>
262</html>
263`))
264