1// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//    http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// run_testlist is a tool runs a dEQP test list, using multiple sand-boxed
16// processes.
17//
18// Unlike simply running deqp with its --deqp-caselist-file flag, run_testlist
19// uses multiple sand-boxed processes, which greatly reduces testing time, and
20// gracefully handles crashing processes.
21package main
22
23import (
24	"bytes"
25	"encoding/json"
26	"errors"
27	"flag"
28	"fmt"
29	"io/ioutil"
30	"log"
31	"math/rand"
32	"os"
33	"path/filepath"
34	"regexp"
35	"runtime"
36	"strings"
37	"time"
38
39	"../../cause"
40	"../../cov"
41	"../../deqp"
42	"../../llvm"
43	"../../shell"
44	"../../testlist"
45	"../../util"
46)
47
48func min(a, b int) int {
49	if a < b {
50		return a
51	}
52	return b
53}
54
55var (
56	deqpVkBinary     = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
57	testList         = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
58	numThreads       = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
59	maxProcMemory    = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
60	output           = flag.String("output", "results.json", "path to an output JSON results file")
61	filter           = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
62	limit            = flag.Int("limit", 0, "only run a maximum of this number of tests")
63	shuffle          = flag.Bool("shuffle", false, "shuffle tests")
64	noResults        = flag.Bool("no-results", false, "disable generation of results.json file")
65	genCoverage      = flag.Bool("coverage", false, "generate test coverage")
66	enableValidation = flag.Bool("validation", false, "run deqp-vk with Vulkan validation layers")
67)
68
69const testTimeout = time.Minute * 2
70
71func run() error {
72	group := testlist.Group{
73		Name: "",
74		File: *testList,
75		API:  testlist.Vulkan,
76	}
77	if err := group.Load(); err != nil {
78		return err
79	}
80
81	if *filter != "" {
82		if strings.HasPrefix(*filter, "/") {
83			re := regexp.MustCompile((*filter)[1:])
84			group = group.Filter(re.MatchString)
85		} else {
86			group = group.Filter(func(name string) bool {
87				ok, _ := filepath.Match(*filter, name)
88				return ok
89			})
90		}
91	}
92
93	shell.MaxProcMemory = *maxProcMemory
94
95	if *shuffle {
96		rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
97		rnd.Shuffle(len(group.Tests), func(i, j int) { group.Tests[i], group.Tests[j] = group.Tests[j], group.Tests[i] })
98	}
99
100	if *limit != 0 && len(group.Tests) > *limit {
101		group.Tests = group.Tests[:*limit]
102	}
103
104	log.Printf("Running %d tests...\n", len(group.Tests))
105
106	config := deqp.Config{
107		ExeEgl:           "",
108		ExeGles2:         "",
109		ExeGles3:         "",
110		ExeVulkan:        *deqpVkBinary,
111		Env:              os.Environ(),
112		NumParallelTests: *numThreads,
113		TestLists:        testlist.Lists{group},
114		TestTimeout:      testTimeout,
115		ValidationLayer:  *enableValidation,
116	}
117
118	if *genCoverage {
119		icdPath := findSwiftshaderICD()
120		t := findToolchain(icdPath)
121		config.CoverageEnv = &cov.Env{
122			LLVM:     t.llvm,
123			TurboCov: t.turbocov,
124			RootDir:  projectRootDir(),
125			ExePath:  findSwiftshaderSO(icdPath),
126		}
127	}
128
129	res, err := config.Run()
130	if err != nil {
131		return err
132	}
133
134	counts := map[testlist.Status]int{}
135	for _, r := range res.Tests {
136		counts[r.Status] = counts[r.Status] + 1
137	}
138	for _, s := range testlist.Statuses {
139		if count := counts[s]; count > 0 {
140			log.Printf("%s: %d\n", string(s), count)
141		}
142	}
143
144	if *genCoverage {
145		f, err := os.Create("coverage.dat")
146		if err != nil {
147			return cause.Wrap(err, "Couldn't open coverage.dat file")
148		}
149		if err := res.Coverage.Encode("master", f); err != nil {
150			return cause.Wrap(err, "Couldn't encode coverage data")
151		}
152	}
153
154	if !*noResults {
155		err = res.Save(*output)
156		if err != nil {
157			return err
158		}
159	}
160
161	return nil
162}
163
164func findSwiftshaderICD() string {
165	icdPaths := strings.Split(os.Getenv("VK_ICD_FILENAMES"), ";")
166	for _, icdPath := range icdPaths {
167		_, file := filepath.Split(icdPath)
168		if file == "vk_swiftshader_icd.json" {
169			return icdPath
170		}
171	}
172	panic("Cannot find vk_swiftshader_icd.json in VK_ICD_FILENAMES")
173}
174
175func findSwiftshaderSO(vkSwiftshaderICD string) string {
176	root := struct {
177		ICD struct {
178			Path string `json:"library_path"`
179		}
180	}{}
181
182	icd, err := ioutil.ReadFile(vkSwiftshaderICD)
183	if err != nil {
184		panic(fmt.Errorf("Could not read '%v'. %v", vkSwiftshaderICD, err))
185	}
186
187	if err := json.NewDecoder(bytes.NewReader(icd)).Decode(&root); err != nil {
188		panic(fmt.Errorf("Could not parse '%v'. %v", vkSwiftshaderICD, err))
189	}
190
191	if util.IsFile(root.ICD.Path) {
192		return root.ICD.Path
193	}
194	dir := filepath.Dir(vkSwiftshaderICD)
195	path, err := filepath.Abs(filepath.Join(dir, root.ICD.Path))
196	if err != nil {
197		panic(fmt.Errorf("Could not locate ICD so at '%v'. %v", root.ICD.Path, err))
198	}
199
200	return path
201}
202
203type toolchain struct {
204	llvm     llvm.Toolchain
205	turbocov string
206}
207
208func findToolchain(vkSwiftshaderICD string) toolchain {
209	minVersion := llvm.Version{Major: 7}
210
211	// Try finding the llvm toolchain via the CMake generated
212	// coverage-toolchain.txt file that sits next to vk_swiftshader_icd.json.
213	dir := filepath.Dir(vkSwiftshaderICD)
214	toolchainInfoPath := filepath.Join(dir, "coverage-toolchain.txt")
215	if util.IsFile(toolchainInfoPath) {
216		if file, err := os.Open(toolchainInfoPath); err == nil {
217			defer file.Close()
218			content := struct {
219				LLVM     string `json:"llvm"`
220				TurboCov string `json:"turbo-cov"`
221			}{}
222			err := json.NewDecoder(file).Decode(&content)
223			if err != nil {
224				log.Fatalf("Couldn't read 'toolchainInfoPath': %v", err)
225			}
226			if t := llvm.Search(content.LLVM).FindAtLeast(minVersion); t != nil {
227				return toolchain{*t, content.TurboCov}
228			}
229		}
230	}
231
232	// Fallback, try searching PATH.
233	if t := llvm.Search().FindAtLeast(minVersion); t != nil {
234		return toolchain{*t, ""}
235	}
236
237	log.Fatal("Could not find LLVM toolchain")
238	return toolchain{}
239}
240
241func projectRootDir() string {
242	_, thisFile, _, _ := runtime.Caller(1)
243	thisDir := filepath.Dir(thisFile)
244	root, err := filepath.Abs(filepath.Join(thisDir, "../../../.."))
245	if err != nil {
246		panic(err)
247	}
248	return root
249}
250
251func main() {
252	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
253	flag.Parse()
254	if err := run(); err != nil {
255		_, _ = fmt.Fprintln(os.Stderr, err)
256		os.Exit(-1)
257	}
258}
259