1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# Vulkan CTS
5# ----------
6#
7# Copyright (c) 2016 Google Inc.
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import re
25import sys
26
27from fnmatch import fnmatch
28
29sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts"))
30sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "scripts", "log"))
31
32from build.common import readFile
33from log_parser import StatusCode, BatchResultParser
34
35ALLOWED_STATUS_CODES = set([
36		StatusCode.PASS,
37		StatusCode.NOT_SUPPORTED,
38		StatusCode.QUALITY_WARNING,
39		StatusCode.COMPATIBILITY_WARNING
40	])
41
42STATEMENT_PATTERN	= "STATEMENT-*"
43TEST_LOG_PATTERN	= "*.qpa"
44GIT_STATUS_PATTERN	= "git-status.txt"
45GIT_LOG_PATTERN		= "git-log.txt"
46PATCH_PATTERN		= "*.patch"
47
48class PackageDescription:
49	def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, otherItems):
50		self.basePath		= basePath
51		self.statement		= statement
52		self.testLogs		= testLogs
53		self.gitStatus		= gitStatus
54		self.gitLog			= gitLog
55		self.patches		= patches
56		self.otherItems		= otherItems
57
58class ValidationMessage:
59	TYPE_ERROR		= 0
60	TYPE_WARNING	= 1
61
62	def __init__ (self, type, filename, message):
63		self.type		= type
64		self.filename	= filename
65		self.message	= message
66
67	def __str__ (self):
68		prefix = {self.TYPE_ERROR: "ERROR: ", self.TYPE_WARNING: "WARNING: "}
69		return prefix[self.type] + os.path.basename(self.filename) + ": " + self.message
70
71def error (filename, message):
72	return ValidationMessage(ValidationMessage.TYPE_ERROR, filename, message)
73
74def warning (filename, message):
75	return ValidationMessage(ValidationMessage.TYPE_WARNING, filename, message)
76
77def getPackageDescription (packagePath):
78	allItems	= os.listdir(packagePath)
79	statement	= None
80	testLogs	= []
81	gitStatus	= None
82	gitLog		= None
83	patches		= []
84	otherItems	= []
85
86	for item in allItems:
87		if fnmatch(item, STATEMENT_PATTERN):
88			assert statement == None
89			statement = item
90		elif fnmatch(item, TEST_LOG_PATTERN):
91			testLogs.append(item)
92		elif fnmatch(item, GIT_STATUS_PATTERN):
93			assert gitStatus == None
94			gitStatus = item
95		elif fnmatch(item, GIT_LOG_PATTERN):
96			assert gitLog == None
97			gitLog = item
98		elif fnmatch(item, PATCH_PATTERN):
99			patches.append(item)
100		else:
101			otherItems.append(item)
102
103	return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, otherItems)
104
105def readMustpass (filename):
106	f = open(filename, 'rb')
107	cases = []
108	for line in f:
109		s = line.strip()
110		if len(s) > 0:
111			cases.append(s)
112	return cases
113
114def readTestLog (filename):
115	parser = BatchResultParser()
116	return parser.parseFile(filename)
117
118def verifyTestLog (filename, mustpass):
119	results			= readTestLog(filename)
120	messages			= []
121	resultOrderOk	= True
122
123	# Mustpass case names must be unique
124	assert len(mustpass) == len(set(mustpass))
125
126	# Verify number of results
127	if len(results) != len(mustpass):
128		messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))
129
130	caseNameToResultNdx = {}
131	for ndx in xrange(len(results)):
132		result = results[ndx]
133		if not result in caseNameToResultNdx:
134			caseNameToResultNdx[result.name] = ndx
135		else:
136			messages.append(error(filename, "Multiple results for " + result.name))
137
138	# Verify that all results are present and valid
139	for ndx in xrange(len(mustpass)):
140		caseName = mustpass[ndx]
141
142		if caseName in caseNameToResultNdx:
143			resultNdx	= caseNameToResultNdx[caseName]
144			result		= results[resultNdx]
145
146			if resultNdx != ndx:
147				resultOrderOk = False
148
149			if not result.statusCode in ALLOWED_STATUS_CODES:
150				messages.append(error(filename, result.name + ": " + result.statusCode))
151		else:
152			messages.append(error(filename, "Missing result for " + caseName))
153
154	if len(results) == len(mustpass) and not resultOrderOk:
155		messages.append(error(filename, "Results are not in the expected order"))
156
157	return messages
158
159def beginsWith (str, prefix):
160	return str[:len(prefix)] == prefix
161
162def verifyStatement (package):
163	messages	= []
164
165	if package.statement != None:
166		statementPath	= os.path.join(package.basePath, package.statement)
167		statement		= readFile(statementPath)
168		hasVersion		= False
169		hasProduct		= False
170		hasCpu			= False
171		hasOs			= False
172
173		for line in statement.splitlines():
174			if beginsWith(line, "CONFORM_VERSION:"):
175				if hasVersion:
176					messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
177				else:
178					hasVersion = True
179			elif beginsWith(line, "PRODUCT:"):
180				hasProduct = True # Multiple products allowed
181			elif beginsWith(line, "CPU:"):
182				if hasCpu:
183					messages.append(error(statementPath, "Multiple PRODUCTs"))
184				else:
185					hasCpu = True
186			elif beginsWith(line, "OS:"):
187				if hasOs:
188					messages.append(error(statementPath, "Multiple OSes"))
189				else:
190					hasOs = True
191
192		if not hasVersion:
193			messages.append(error(statementPath, "No CONFORM_VERSION"))
194		if not hasProduct:
195			messages.append(error(statementPath, "No PRODUCT"))
196		if not hasCpu:
197			messages.append(error(statementPath, "No CPU"))
198		if not hasOs:
199			messages.append(error(statementPath, "No OS"))
200	else:
201		messages.append(error(package.basePath, "Missing conformance statement file"))
202
203	return messages
204
205def verifyGitStatus (package):
206	messages = []
207
208	if package.gitStatus != None:
209		statusPath	= os.path.join(package.basePath, package.gitStatus)
210		status		= readFile(statusPath)
211
212		if status.find("nothing to commit, working directory clean") < 0:
213			messages.append(error(package.basePath, "Working directory is not clean"))
214	else:
215		messages.append(error(package.basePath, "Missing git-status.txt"))
216
217	return messages
218
219def isGitLogEmpty (package):
220	assert package.gitLog != None
221
222	logPath	= os.path.join(package.basePath, package.gitLog)
223	log		= readFile(logPath)
224
225	return len(log.strip()) == 0
226
227def verifyGitLog (package):
228	messages = []
229
230	if package.gitLog != None:
231		if not isGitLogEmpty(package):
232			messages.append(warning(os.path.join(package.basePath, package.gitLog), "Log is not empty"))
233	else:
234		messages.append(error(package.basePath, "Missing git-log.txt"))
235
236	return messages
237
238def verifyPatches (package):
239	messages	= []
240	hasPatches	= len(package.patches)
241	logEmpty	= package.gitLog and isGitLogEmpty(package)
242
243	if hasPatches and logEmpty:
244		messages.append(error(package.basePath, "Package includes patches but log is empty"))
245	elif not hasPatches and not logEmpty:
246		messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))
247
248	return messages
249
250def verifyTestLogs (package, mustpass):
251	messages	= []
252
253	for testLogFile in package.testLogs:
254		messages += verifyTestLog(os.path.join(package.basePath, testLogFile), mustpass)
255
256	if len(package.testLogs) == 0:
257		messages.append(error(package.basePath, "No test log files found"))
258
259	return messages
260
261def verifyPackage (package, mustpass):
262	messages = []
263
264	messages += verifyStatement(package)
265	messages += verifyGitStatus(package)
266	messages += verifyGitLog(package)
267	messages += verifyPatches(package)
268	messages += verifyTestLogs(package, mustpass)
269
270	for item in package.otherItems:
271		messages.append(warning(os.path.join(package.basePath, item), "Unknown file"))
272
273	return messages
274
275if __name__ == "__main__":
276	if len(sys.argv) != 3:
277		print "%s: [extracted submission package] [mustpass]" % sys.argv[0]
278		sys.exit(-1)
279
280	packagePath		= os.path.normpath(sys.argv[1])
281	mustpassPath	= sys.argv[2]
282	package			= getPackageDescription(packagePath)
283	mustpass		= readMustpass(mustpassPath)
284	messages		= verifyPackage(package, mustpass)
285
286	errors			= [m for m in messages if m.type == ValidationMessage.TYPE_ERROR]
287	warnings		= [m for m in messages if m.type == ValidationMessage.TYPE_WARNING]
288
289	for message in messages:
290		print str(message)
291
292	print ""
293
294	if len(errors) > 0:
295		print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings))
296		sys.exit(-2)
297	elif len(warnings) > 0:
298		print "Found %d warnings, manual review required" % len(warnings)
299		sys.exit(-1)
300	else:
301		print "All validation checks passed"
302