1/* Copyright (c) 2015, Google Inc.
2 *
3 * Permission to use, copy, modify, and/or distribute this software for any
4 * purpose with or without fee is hereby granted, provided that the above
5 * copyright notice and this permission notice appear in all copies.
6 *
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
14
15package main
16
17import (
18	"bufio"
19	"bytes"
20	"encoding/json"
21	"errors"
22	"flag"
23	"fmt"
24	"math/rand"
25	"os"
26	"os/exec"
27	"path"
28	"runtime"
29	"strconv"
30	"strings"
31	"sync"
32	"syscall"
33
34	"boringssl.googlesource.com/boringssl/util/testresult"
35)
36
37// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
38
39var (
40	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
41	useCallgrind    = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.")
42	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
43	useSDE          = flag.Bool("sde", false, "If true, run BoringSSL code under Intel's SDE for each supported chip")
44	sdePath         = flag.String("sde-path", "sde", "The path to find the sde binary.")
45	buildDir        = flag.String("build-dir", "build", "The build directory to run the tests from.")
46	numWorkers      = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers when testing.")
47	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
48	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
49	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
50	simulateARMCPUs = flag.Bool("simulate-arm-cpus", simulateARMCPUsDefault(), "If true, runs tests simulating different ARM CPUs.")
51)
52
53func simulateARMCPUsDefault() bool {
54	return runtime.GOOS == "linux" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64")
55}
56
57type test struct {
58	args             []string
59	shard, numShards int
60	// cpu, if not empty, contains a code to simulate. For SDE, run `sde64
61	// -help` to get a list of these codes. For ARM, see gtest_main.cc for
62	// the supported values.
63	cpu string
64}
65
66type result struct {
67	Test   test
68	Passed bool
69	Error  error
70}
71
72// sdeCPUs contains a list of CPU code that we run all tests under when *useSDE
73// is true.
74var sdeCPUs = []string{
75	"p4p", // Pentium4 Prescott
76	"mrm", // Merom
77	"pnr", // Penryn
78	"nhm", // Nehalem
79	"wsm", // Westmere
80	"snb", // Sandy Bridge
81	"ivb", // Ivy Bridge
82	"hsw", // Haswell
83	"bdw", // Broadwell
84	"skx", // Skylake Server
85	"skl", // Skylake Client
86	"cnl", // Cannonlake
87	"knl", // Knights Landing
88	"slt", // Saltwell
89	"slm", // Silvermont
90	"glm", // Goldmont
91	"knm", // Knights Mill
92}
93
94var armCPUs = []string{
95	"none",   // No support for any ARM extensions.
96	"neon",   // Support for NEON.
97	"crypto", // Support for NEON and crypto extensions.
98}
99
100func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
101	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"}
102	if dbAttach {
103		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
104	}
105	valgrindArgs = append(valgrindArgs, path)
106	valgrindArgs = append(valgrindArgs, args...)
107
108	return exec.Command("valgrind", valgrindArgs...)
109}
110
111func callgrindOf(path string, args ...string) *exec.Cmd {
112	valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"}
113	valgrindArgs = append(valgrindArgs, path)
114	valgrindArgs = append(valgrindArgs, args...)
115
116	return exec.Command("valgrind", valgrindArgs...)
117}
118
119func gdbOf(path string, args ...string) *exec.Cmd {
120	xtermArgs := []string{"-e", "gdb", "--args"}
121	xtermArgs = append(xtermArgs, path)
122	xtermArgs = append(xtermArgs, args...)
123
124	return exec.Command("xterm", xtermArgs...)
125}
126
127func sdeOf(cpu, path string, args ...string) *exec.Cmd {
128	sdeArgs := []string{"-" + cpu}
129	// The kernel's vdso code for gettimeofday sometimes uses the RDTSCP
130	// instruction. Although SDE has a -chip_check_vsyscall flag that
131	// excludes such code by default, it does not seem to work. Instead,
132	// pass the -chip_check_exe_only flag which retains test coverage when
133	// statically linked and excludes the vdso.
134	if cpu == "p4p" || cpu == "pnr" || cpu == "mrm" || cpu == "slt" {
135		sdeArgs = append(sdeArgs, "-chip_check_exe_only")
136	}
137	sdeArgs = append(sdeArgs, "--", path)
138	sdeArgs = append(sdeArgs, args...)
139	return exec.Command(*sdePath, sdeArgs...)
140}
141
142var (
143	errMoreMallocs = errors.New("child process did not exhaust all allocation calls")
144	errTestSkipped = errors.New("test was skipped")
145)
146
147func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
148	prog := path.Join(*buildDir, test.args[0])
149	args := append([]string{}, test.args[1:]...)
150	if *simulateARMCPUs && test.cpu != "" {
151		args = append(args, "--cpu=" + test.cpu)
152	}
153	if *useSDE {
154		// SDE is neither compatible with the unwind tester nor automatically
155		// detected.
156		args = append(args, "--no_unwind_tests")
157	}
158	var cmd *exec.Cmd
159	if *useValgrind {
160		cmd = valgrindOf(false, prog, args...)
161	} else if *useCallgrind {
162		cmd = callgrindOf(prog, args...)
163	} else if *useGDB {
164		cmd = gdbOf(prog, args...)
165	} else if *useSDE {
166		cmd = sdeOf(test.cpu, prog, args...)
167	} else {
168		cmd = exec.Command(prog, args...)
169	}
170	var outBuf bytes.Buffer
171	cmd.Stdout = &outBuf
172	cmd.Stderr = &outBuf
173	if mallocNumToFail >= 0 {
174		cmd.Env = os.Environ()
175		cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
176		if *mallocTestDebug {
177			cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
178		}
179		cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
180	}
181
182	if err := cmd.Start(); err != nil {
183		return false, err
184	}
185	if err := cmd.Wait(); err != nil {
186		if exitError, ok := err.(*exec.ExitError); ok {
187			switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
188			case 88:
189				return false, errMoreMallocs
190			case 89:
191				fmt.Print(string(outBuf.Bytes()))
192				return false, errTestSkipped
193			}
194		}
195		fmt.Print(string(outBuf.Bytes()))
196		return false, err
197	}
198
199	// Account for Windows line-endings.
200	stdout := bytes.Replace(outBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
201
202	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
203		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
204		return true, nil
205	}
206
207	// Also accept a googletest-style pass line. This is left here in
208	// transition until the tests are all converted and this script made
209	// unnecessary.
210	if bytes.Contains(stdout, []byte("\n[  PASSED  ]")) {
211		return true, nil
212	}
213
214	fmt.Print(string(outBuf.Bytes()))
215	return false, nil
216}
217
218func runTest(test test) (bool, error) {
219	if *mallocTest < 0 {
220		return runTestOnce(test, -1)
221	}
222
223	for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
224		if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
225			if err != nil {
226				err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
227			}
228			return passed, err
229		}
230	}
231}
232
233// setWorkingDirectory walks up directories as needed until the current working
234// directory is the top of a BoringSSL checkout.
235func setWorkingDirectory() {
236	for i := 0; i < 64; i++ {
237		if _, err := os.Stat("BUILDING.md"); err == nil {
238			return
239		}
240		os.Chdir("..")
241	}
242
243	panic("Couldn't find BUILDING.md in a parent directory!")
244}
245
246func parseTestConfig(filename string) ([]test, error) {
247	in, err := os.Open(filename)
248	if err != nil {
249		return nil, err
250	}
251	defer in.Close()
252
253	decoder := json.NewDecoder(in)
254	var testArgs [][]string
255	if err := decoder.Decode(&testArgs); err != nil {
256		return nil, err
257	}
258
259	var result []test
260	for _, args := range testArgs {
261		result = append(result, test{args: args})
262	}
263	return result, nil
264}
265
266func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) {
267	defer done.Done()
268	for test := range tests {
269		passed, err := runTest(test)
270		results <- result{test, passed, err}
271	}
272}
273
274func (t test) shortName() string {
275	return t.args[0] + t.shardMsg() + t.cpuMsg()
276}
277
278func (t test) longName() string {
279	return strings.Join(t.args, " ") + t.cpuMsg()
280}
281
282func (t test) shardMsg() string {
283	if t.numShards == 0 {
284		return ""
285	}
286
287	return fmt.Sprintf(" [shard %d/%d]", t.shard+1, t.numShards)
288}
289
290func (t test) cpuMsg() string {
291	if len(t.cpu) == 0 {
292		return ""
293	}
294
295	return fmt.Sprintf(" (for CPU %q)", t.cpu)
296}
297
298func (t test) getGTestShards() ([]test, error) {
299	if *numWorkers == 1 || len(t.args) != 1 {
300		return []test{t}, nil
301	}
302
303	// Only shard the three GTest-based tests.
304	if t.args[0] != "crypto/crypto_test" && t.args[0] != "ssl/ssl_test" && t.args[0] != "decrepit/decrepit_test" {
305		return []test{t}, nil
306	}
307
308	prog := path.Join(*buildDir, t.args[0])
309	cmd := exec.Command(prog, "--gtest_list_tests")
310	var stdout bytes.Buffer
311	cmd.Stdout = &stdout
312	if err := cmd.Start(); err != nil {
313		return nil, err
314	}
315	if err := cmd.Wait(); err != nil {
316		return nil, err
317	}
318
319	var group string
320	var tests []string
321	scanner := bufio.NewScanner(&stdout)
322	for scanner.Scan() {
323		line := scanner.Text()
324
325		// Remove the parameter comment and trailing space.
326		if idx := strings.Index(line, "#"); idx >= 0 {
327			line = line[:idx]
328		}
329		line = strings.TrimSpace(line)
330		if len(line) == 0 {
331			continue
332		}
333
334		if line[len(line)-1] == '.' {
335			group = line
336			continue
337		}
338
339		if len(group) == 0 {
340			return nil, fmt.Errorf("found test case %q without group", line)
341		}
342		tests = append(tests, group+line)
343	}
344
345	const testsPerShard = 20
346	if len(tests) <= testsPerShard {
347		return []test{t}, nil
348	}
349
350	// Slow tests which process large test vector files tend to be grouped
351	// together, so shuffle the order.
352	shuffled := make([]string, len(tests))
353	perm := rand.Perm(len(tests))
354	for i, j := range perm {
355		shuffled[i] = tests[j]
356	}
357
358	var shards []test
359	for i := 0; i < len(shuffled); i += testsPerShard {
360		n := len(shuffled) - i
361		if n > testsPerShard {
362			n = testsPerShard
363		}
364		shard := t
365		shard.args = []string{shard.args[0], "--gtest_filter=" + strings.Join(shuffled[i:i+n], ":")}
366		shard.shard = len(shards)
367		shards = append(shards, shard)
368	}
369
370	for i := range shards {
371		shards[i].numShards = len(shards)
372	}
373
374	return shards, nil
375}
376
377func main() {
378	flag.Parse()
379	setWorkingDirectory()
380
381	testCases, err := parseTestConfig("util/all_tests.json")
382	if err != nil {
383		fmt.Printf("Failed to parse input: %s\n", err)
384		os.Exit(1)
385	}
386
387	var wg sync.WaitGroup
388	tests := make(chan test, *numWorkers)
389	results := make(chan result, *numWorkers)
390
391	for i := 0; i < *numWorkers; i++ {
392		wg.Add(1)
393		go worker(tests, results, &wg)
394	}
395
396	go func() {
397		for _, test := range testCases {
398			if *useSDE {
399				// SDE generates plenty of tasks and gets slower
400				// with additional sharding.
401				for _, cpu := range sdeCPUs {
402					testForCPU := test
403					testForCPU.cpu = cpu
404					tests <- testForCPU
405				}
406			} else if *simulateARMCPUs {
407				// This mode is run instead of the default path,
408				// so also include the native flow.
409				tests <- test
410				for _, cpu := range armCPUs {
411					testForCPU := test
412					testForCPU.cpu = cpu
413					tests <- testForCPU
414				}
415			} else {
416				shards, err := test.getGTestShards()
417				if err != nil {
418					fmt.Printf("Error listing tests: %s\n", err)
419					os.Exit(1)
420				}
421				for _, shard := range shards {
422					tests <- shard
423				}
424			}
425		}
426		close(tests)
427
428		wg.Wait()
429		close(results)
430	}()
431
432	testOutput := testresult.NewResults()
433	var failed, skipped []test
434	for testResult := range results {
435		test := testResult.Test
436		args := test.args
437
438		if testResult.Error == errTestSkipped {
439			fmt.Printf("%s\n", test.longName())
440			fmt.Printf("%s was skipped\n", args[0])
441			skipped = append(skipped, test)
442			testOutput.AddSkip(test.longName())
443		} else if testResult.Error != nil {
444			fmt.Printf("%s\n", test.longName())
445			fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error)
446			failed = append(failed, test)
447			testOutput.AddResult(test.longName(), "CRASH")
448		} else if !testResult.Passed {
449			fmt.Printf("%s\n", test.longName())
450			fmt.Printf("%s failed to print PASS on the last line.\n", args[0])
451			failed = append(failed, test)
452			testOutput.AddResult(test.longName(), "FAIL")
453		} else {
454			fmt.Printf("%s\n", test.shortName())
455			testOutput.AddResult(test.longName(), "PASS")
456		}
457	}
458
459	if *jsonOutput != "" {
460		if err := testOutput.WriteToFile(*jsonOutput); err != nil {
461			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
462		}
463	}
464
465	if len(skipped) > 0 {
466		fmt.Printf("\n%d of %d tests were skipped:\n", len(skipped), len(testCases))
467		for _, test := range skipped {
468			fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg())
469		}
470	}
471
472	if len(failed) > 0 {
473		fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases))
474		for _, test := range failed {
475			fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg())
476		}
477		os.Exit(1)
478	}
479
480	fmt.Printf("\nAll tests passed!\n")
481}
482