1// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//    http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Package testlist provides utilities for handling test lists.
16package testlist
17
18import (
19	"bytes"
20	"crypto/sha1"
21	"encoding/gob"
22	"encoding/hex"
23	"encoding/json"
24	"io/ioutil"
25	"path/filepath"
26	"sort"
27	"strings"
28
29	"../cause"
30)
31
32// API is an enumerator of graphics APIs.
33type API string
34
35// Graphics APIs.
36const (
37	EGL    = API("egl")
38	GLES2  = API("gles2")
39	GLES3  = API("gles3")
40	Vulkan = API("vulkan")
41)
42
43// Group is a list of tests to be run for a single API.
44type Group struct {
45	Name  string
46	File  string
47	API   API
48	Tests []string
49}
50
51// Load loads the test list file and appends all tests to the Group.
52func (g *Group) Load() error {
53	tests, err := ioutil.ReadFile(g.File)
54	if err != nil {
55		return cause.Wrap(err, "Couldn't read '%s'", tests)
56	}
57	for _, line := range strings.Split(string(tests), "\n") {
58		line = strings.TrimSpace(line)
59		if line != "" && !strings.HasPrefix(line, "#") {
60			g.Tests = append(g.Tests, line)
61		}
62	}
63	sort.Strings(g.Tests)
64	return nil
65}
66
67// Filter returns a new Group that contains only tests that match the predicate.
68func (g Group) Filter(pred func(string) bool) Group {
69	out := Group{
70		Name: g.Name,
71		File: g.File,
72		API:  g.API,
73	}
74	for _, test := range g.Tests {
75		if pred(test) {
76			out.Tests = append(out.Tests, test)
77		}
78	}
79	return out
80}
81
82// Limit returns a new Group that contains a maximum of limit tests.
83func (g Group) Limit(limit int) Group {
84	out := Group{
85		Name:  g.Name,
86		File:  g.File,
87		API:   g.API,
88		Tests: g.Tests,
89	}
90	if len(g.Tests) > limit {
91		out.Tests = g.Tests[:limit]
92	}
93	return out
94}
95
96// Lists is the full list of tests to be run.
97type Lists []Group
98
99// Filter returns a new Lists that contains only tests that match the predicate.
100func (l Lists) Filter(pred func(string) bool) Lists {
101	out := Lists{}
102	for _, group := range l {
103		filtered := group.Filter(pred)
104		if len(filtered.Tests) > 0 {
105			out = append(out, filtered)
106		}
107	}
108	return out
109}
110
111// Hash returns a SHA1 hash of the set of tests.
112func (l Lists) Hash() string {
113	h := sha1.New()
114	if err := gob.NewEncoder(h).Encode(l); err != nil {
115		panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
116	}
117	return hex.EncodeToString(h.Sum(nil))
118}
119
120// Load loads the test list json file and returns the full set of tests.
121func Load(root, jsonPath string) (Lists, error) {
122	root, err := filepath.Abs(root)
123	if err != nil {
124		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
125	}
126
127	jsonPath, err = filepath.Abs(jsonPath)
128	if err != nil {
129		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
130	}
131
132	i, err := ioutil.ReadFile(jsonPath)
133	if err != nil {
134		return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
135	}
136
137	var jsonGroups []struct {
138		Name     string
139		API      string
140		TestFile string `json:"tests"`
141	}
142	if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
143		return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
144	}
145
146	dir := filepath.Dir(jsonPath)
147
148	out := make(Lists, len(jsonGroups))
149	for i, jsonGroup := range jsonGroups {
150		group := Group{
151			Name: jsonGroup.Name,
152			File: filepath.Join(dir, jsonGroup.TestFile),
153			API:  API(jsonGroup.API),
154		}
155		if err := group.Load(); err != nil {
156			return nil, err
157		}
158
159		// Make the path relative before displaying it to the world.
160		relPath, err := filepath.Rel(root, group.File)
161		if err != nil {
162			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
163		}
164		group.File = relPath
165
166		out[i] = group
167	}
168
169	return out, nil
170}
171
172// Status is an enumerator of test results.
173type Status string
174
175const (
176	// Pass is the status of a successful test.
177	Pass = Status("PASS")
178	// Fail is the status of a failed test.
179	Fail = Status("FAIL")
180	// Timeout is the status of a test that failed to complete in the alloted
181	// time.
182	Timeout = Status("TIMEOUT")
183	// Crash is the status of a test that crashed.
184	Crash = Status("CRASH")
185	// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
186	Unimplemented = Status("UNIMPLEMENTED")
187	// Unsupported is the status of a test that failed with UNSUPPORTED().
188	Unsupported = Status("UNSUPPORTED")
189	// Unreachable is the status of a test that failed with UNREACHABLE().
190	Unreachable = Status("UNREACHABLE")
191	// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
192	Assert = Status("ASSERT")
193	// Abort is the status of a test that failed with ABORT().
194	Abort = Status("ABORT")
195	// NotSupported is the status of a test feature not supported by the driver.
196	NotSupported = Status("NOT_SUPPORTED")
197	// CompatibilityWarning is the status passing test with a warning.
198	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
199	// QualityWarning is the status passing test with a warning.
200	QualityWarning = Status("QUALITY_WARNING")
201	// InternalError is the status of a test that failed on an API usage error.
202	InternalError = Status("INTERNAL_ERROR")
203)
204
205// Statuses is the full list of status types
206var Statuses = []Status{
207	Pass,
208	Fail,
209	Timeout,
210	Crash,
211	Unimplemented,
212	Unsupported,
213	Unreachable,
214	Assert,
215	Abort,
216	NotSupported,
217	CompatibilityWarning,
218	QualityWarning,
219	InternalError,
220}
221
222// Failing returns true if the task status requires fixing.
223func (s Status) Failing() bool {
224	switch s {
225	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
226		return true
227	case Unsupported:
228		// This may seem surprising that this should be a failure, however these
229		// should not be reached, as dEQP should not be using features that are
230		// not advertised.
231		return true
232	default:
233		return false
234	}
235}
236
237// Passing returns true if the task status is considered a pass.
238func (s Status) Passing() bool {
239	switch s {
240	case Pass, CompatibilityWarning, QualityWarning:
241		return true
242	default:
243		return false
244	}
245}
246
247// FilePathWithStatus returns the path to the test list file with the status
248// appended before the file extension.
249func FilePathWithStatus(listPath string, status Status) string {
250	ext := filepath.Ext(listPath)
251	name := listPath[:len(listPath)-len(ext)]
252	return name + "-" + string(status) + ext
253}
254