1 /*-------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2015 Google Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Vulkan Test Package
22 *//*--------------------------------------------------------------------*/
23
24 #include "vktTestPackage.hpp"
25
26 #include "tcuPlatform.hpp"
27 #include "tcuTestCase.hpp"
28 #include "tcuTestLog.hpp"
29 #include "tcuCommandLine.hpp"
30
31 #include "vkPlatform.hpp"
32 #include "vkPrograms.hpp"
33 #include "vkBinaryRegistry.hpp"
34 #include "vkGlslToSpirV.hpp"
35 #include "vkDebugReportUtil.hpp"
36 #include "vkQueryUtil.hpp"
37
38 #include "deUniquePtr.hpp"
39
40 #include "vktTestGroupUtil.hpp"
41 #include "vktApiTests.hpp"
42 #include "vktPipelineTests.hpp"
43 #include "vktBindingModelTests.hpp"
44 #include "vktSpvAsmTests.hpp"
45 #include "vktShaderLibrary.hpp"
46 #include "vktRenderPassTests.hpp"
47 #include "vktMemoryTests.hpp"
48 #include "vktShaderRenderDiscardTests.hpp"
49 #include "vktShaderRenderIndexingTests.hpp"
50 #include "vktShaderRenderLoopTests.hpp"
51 #include "vktShaderRenderMatrixTests.hpp"
52 #include "vktShaderRenderOperatorTests.hpp"
53 #include "vktShaderRenderReturnTests.hpp"
54 #include "vktShaderRenderStructTests.hpp"
55 #include "vktShaderRenderSwitchTests.hpp"
56 #include "vktShaderBuiltinTests.hpp"
57 #include "vktOpaqueTypeIndexingTests.hpp"
58 #include "vktUniformBlockTests.hpp"
59 #include "vktDynamicStateTests.hpp"
60 #include "vktSSBOLayoutTests.hpp"
61 #include "vktQueryPoolTests.hpp"
62 #include "vktDrawTests.hpp"
63 #include "vktComputeTests.hpp"
64 #include "vktImageTests.hpp"
65 #include "vktInfoTests.hpp"
66 #include "vktWsiTests.hpp"
67 #include "vktSynchronization.hpp"
68 #include "vktSparseResourcesTests.hpp"
69
70 #include <vector>
71 #include <sstream>
72
73 namespace // compilation
74 {
75
compileProgram(const glu::ProgramSources & source,glu::ShaderProgramInfo * buildInfo)76 vk::ProgramBinary* compileProgram (const glu::ProgramSources& source, glu::ShaderProgramInfo* buildInfo)
77 {
78 return vk::buildProgram(source, vk::PROGRAM_FORMAT_SPIRV, buildInfo);
79 }
80
compileProgram(const vk::SpirVAsmSource & source,vk::SpirVProgramInfo * buildInfo)81 vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVProgramInfo* buildInfo)
82 {
83 return vk::assembleProgram(source, buildInfo);
84 }
85
86 template <typename InfoType, typename IteratorType>
buildProgram(const std::string & casePath,IteratorType iter,const vk::BinaryRegistryReader & prebuiltBinRegistry,tcu::TestLog & log,vk::BinaryCollection * progCollection)87 vk::ProgramBinary* buildProgram (const std::string& casePath,
88 IteratorType iter,
89 const vk::BinaryRegistryReader& prebuiltBinRegistry,
90 tcu::TestLog& log,
91 vk::BinaryCollection* progCollection)
92 {
93 const vk::ProgramIdentifier progId (casePath, iter.getName());
94 const tcu::ScopedLogSection progSection (log, iter.getName(), "Program: " + iter.getName());
95 de::MovePtr<vk::ProgramBinary> binProg;
96 InfoType buildInfo;
97
98 try
99 {
100 binProg = de::MovePtr<vk::ProgramBinary>(compileProgram(iter.getProgram(), &buildInfo));
101 log << buildInfo;
102 }
103 catch (const tcu::NotSupportedError& err)
104 {
105 // Try to load from cache
106 log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage;
107
108 binProg = de::MovePtr<vk::ProgramBinary>(prebuiltBinRegistry.loadProgram(progId));
109
110 log << iter.getProgram();
111 }
112 catch (const tcu::Exception&)
113 {
114 // Build failed for other reason
115 log << buildInfo;
116 throw;
117 }
118
119 TCU_CHECK_INTERNAL(binProg);
120
121 {
122 vk::ProgramBinary* const returnBinary = binProg.get();
123
124 progCollection->add(progId.programName, binProg);
125
126 return returnBinary;
127 }
128 }
129
130 } // anonymous(compilation)
131
132 namespace vkt
133 {
134
135 using std::vector;
136 using de::UniquePtr;
137 using de::MovePtr;
138 using tcu::TestLog;
139
140 namespace
141 {
142
createDebugReportRecorder(const vk::PlatformInterface & vkp,const vk::InstanceInterface & vki,vk::VkInstance instance)143 MovePtr<vk::DebugReportRecorder> createDebugReportRecorder (const vk::PlatformInterface& vkp, const vk::InstanceInterface& vki, vk::VkInstance instance)
144 {
145 if (isDebugReportSupported(vkp))
146 return MovePtr<vk::DebugReportRecorder>(new vk::DebugReportRecorder(vki, instance));
147 else
148 TCU_THROW(NotSupportedError, "VK_EXT_debug_report is not supported");
149 }
150
151 } // anonymous
152
153 // TestCaseExecutor
154
155 class TestCaseExecutor : public tcu::TestCaseExecutor
156 {
157 public:
158 TestCaseExecutor (tcu::TestContext& testCtx);
159 ~TestCaseExecutor (void);
160
161 virtual void init (tcu::TestCase* testCase, const std::string& path);
162 virtual void deinit (tcu::TestCase* testCase);
163
164 virtual tcu::TestNode::IterateResult iterate (tcu::TestCase* testCase);
165
166 private:
167 vk::BinaryCollection m_progCollection;
168 vk::BinaryRegistryReader m_prebuiltBinRegistry;
169
170 const UniquePtr<vk::Library> m_library;
171 Context m_context;
172
173 const UniquePtr<vk::DebugReportRecorder> m_debugReportRecorder;
174
175 TestInstance* m_instance; //!< Current test case instance
176 };
177
createLibrary(tcu::TestContext & testCtx)178 static MovePtr<vk::Library> createLibrary (tcu::TestContext& testCtx)
179 {
180 return MovePtr<vk::Library>(testCtx.getPlatform().getVulkanPlatform().createLibrary());
181 }
182
TestCaseExecutor(tcu::TestContext & testCtx)183 TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx)
184 : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt")
185 , m_library (createLibrary(testCtx))
186 , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection)
187 , m_debugReportRecorder (testCtx.getCommandLine().isValidationEnabled()
188 ? createDebugReportRecorder(m_library->getPlatformInterface(),
189 m_context.getInstanceInterface(),
190 m_context.getInstance())
191 : MovePtr<vk::DebugReportRecorder>(DE_NULL))
192 , m_instance (DE_NULL)
193 {
194 }
195
~TestCaseExecutor(void)196 TestCaseExecutor::~TestCaseExecutor (void)
197 {
198 delete m_instance;
199 }
200
init(tcu::TestCase * testCase,const std::string & casePath)201 void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePath)
202 {
203 const TestCase* vktCase = dynamic_cast<TestCase*>(testCase);
204 tcu::TestLog& log = m_context.getTestContext().getLog();
205 vk::SourceCollections sourceProgs;
206
207 DE_UNREF(casePath); // \todo [2015-03-13 pyry] Use this to identify ProgramCollection storage path
208
209 if (!vktCase)
210 TCU_THROW(InternalError, "Test node not an instance of vkt::TestCase");
211
212 m_progCollection.clear();
213 vktCase->initPrograms(sourceProgs);
214
215 for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter)
216 {
217 vk::ProgramBinary* binProg = buildProgram<glu::ShaderProgramInfo, vk::GlslSourceCollection::Iterator>(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection);
218
219 try
220 {
221 std::ostringstream disasm;
222
223 vk::disassembleProgram(*binProg, &disasm);
224
225 log << vk::SpirVAsmSource(disasm.str());
226 }
227 catch (const tcu::NotSupportedError& err)
228 {
229 log << err;
230 }
231 }
232
233 for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator)
234 {
235 buildProgram<vk::SpirVProgramInfo, vk::SpirVAsmCollection::Iterator>(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection);
236 }
237
238 DE_ASSERT(!m_instance);
239 m_instance = vktCase->createInstance(m_context);
240 }
241
deinit(tcu::TestCase *)242 void TestCaseExecutor::deinit (tcu::TestCase*)
243 {
244 delete m_instance;
245 m_instance = DE_NULL;
246
247 // Collect and report any debug messages
248 if (m_debugReportRecorder)
249 {
250 // \note We are not logging INFORMATION and DEBUG messages
251 static const vk::VkDebugReportFlagsEXT errorFlags = vk::VK_DEBUG_REPORT_ERROR_BIT_EXT;
252 static const vk::VkDebugReportFlagsEXT logFlags = errorFlags
253 | vk::VK_DEBUG_REPORT_WARNING_BIT_EXT
254 | vk::VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
255
256 typedef vk::DebugReportRecorder::MessageList DebugMessages;
257
258 const DebugMessages& messages = m_debugReportRecorder->getMessages();
259 tcu::TestLog& log = m_context.getTestContext().getLog();
260
261 if (messages.begin() != messages.end())
262 {
263 const tcu::ScopedLogSection section (log, "DebugMessages", "Debug Messages");
264 int numErrors = 0;
265
266 for (DebugMessages::const_iterator curMsg = messages.begin(); curMsg != messages.end(); ++curMsg)
267 {
268 if ((curMsg->flags & logFlags) != 0)
269 log << tcu::TestLog::Message << *curMsg << tcu::TestLog::EndMessage;
270
271 if ((curMsg->flags & errorFlags) != 0)
272 numErrors += 1;
273 }
274
275 m_debugReportRecorder->clearMessages();
276
277 if (numErrors > 0)
278 m_context.getTestContext().setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, (de::toString(numErrors) + " API usage errors found").c_str());
279 }
280 }
281 }
282
iterate(tcu::TestCase *)283 tcu::TestNode::IterateResult TestCaseExecutor::iterate (tcu::TestCase*)
284 {
285 DE_ASSERT(m_instance);
286
287 const tcu::TestStatus result = m_instance->iterate();
288
289 if (result.isComplete())
290 {
291 // Vulkan tests shouldn't set result directly
292 DE_ASSERT(m_context.getTestContext().getTestResult() == QP_TEST_RESULT_LAST);
293 m_context.getTestContext().setTestResult(result.getCode(), result.getDescription().c_str());
294 return tcu::TestNode::STOP;
295 }
296 else
297 return tcu::TestNode::CONTINUE;
298 }
299
300 // GLSL shader tests
301
createGlslTests(tcu::TestCaseGroup * glslTests)302 void createGlslTests (tcu::TestCaseGroup* glslTests)
303 {
304 tcu::TestContext& testCtx = glslTests->getTestContext();
305
306 // ShaderLibrary-based tests
307 static const struct
308 {
309 const char* name;
310 const char* description;
311 } s_es310Tests[] =
312 {
313 { "arrays", "Arrays" },
314 { "conditionals", "Conditional statements" },
315 { "constant_expressions", "Constant expressions" },
316 { "constants", "Constants" },
317 { "conversions", "Type conversions" },
318 { "functions", "Functions" },
319 { "linkage", "Linking" },
320 { "scoping", "Scoping" },
321 { "swizzles", "Swizzles" },
322 };
323
324 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_es310Tests); ndx++)
325 glslTests->addChild(createShaderLibraryGroup(testCtx,
326 s_es310Tests[ndx].name,
327 s_es310Tests[ndx].description,
328 std::string("vulkan/glsl/es310/") + s_es310Tests[ndx].name + ".test").release());
329
330 // ShaderRenderCase-based tests
331 glslTests->addChild(sr::createDiscardTests (testCtx));
332 glslTests->addChild(sr::createIndexingTests (testCtx));
333 glslTests->addChild(sr::createLoopTests (testCtx));
334 glslTests->addChild(sr::createMatrixTests (testCtx));
335 glslTests->addChild(sr::createOperatorTests (testCtx));
336 glslTests->addChild(sr::createReturnTests (testCtx));
337 glslTests->addChild(sr::createStructTests (testCtx));
338 glslTests->addChild(sr::createSwitchTests (testCtx));
339
340 // ShaderExecutor-based tests
341 glslTests->addChild(shaderexecutor::createBuiltinTests (testCtx));
342 glslTests->addChild(shaderexecutor::createOpaqueTypeIndexingTests (testCtx));
343 }
344
345 // TestPackage
346
TestPackage(tcu::TestContext & testCtx)347 TestPackage::TestPackage (tcu::TestContext& testCtx)
348 : tcu::TestPackage(testCtx, "dEQP-VK", "dEQP Vulkan Tests")
349 {
350 }
351
~TestPackage(void)352 TestPackage::~TestPackage (void)
353 {
354 }
355
createExecutor(void) const356 tcu::TestCaseExecutor* TestPackage::createExecutor (void) const
357 {
358 return new TestCaseExecutor(m_testCtx);
359 }
360
init(void)361 void TestPackage::init (void)
362 {
363 addChild(createTestGroup (m_testCtx, "info", "Build and Device Info Tests", createInfoTests));
364 addChild(api::createTests (m_testCtx));
365 addChild(pipeline::createTests (m_testCtx));
366 addChild(BindingModel::createTests (m_testCtx));
367 addChild(SpirVAssembly::createTests (m_testCtx));
368 addChild(createTestGroup (m_testCtx, "glsl", "GLSL shader execution tests", createGlslTests));
369 addChild(createRenderPassTests (m_testCtx));
370 addChild(memory::createTests (m_testCtx));
371 addChild(ubo::createTests (m_testCtx));
372 addChild(DynamicState::createTests (m_testCtx));
373 addChild(ssbo::createTests (m_testCtx));
374 addChild(QueryPool::createTests (m_testCtx));
375 addChild(Draw::createTests (m_testCtx));
376 addChild(compute::createTests (m_testCtx));
377 addChild(image::createTests (m_testCtx));
378 addChild(wsi::createTests (m_testCtx));
379 addChild(createSynchronizationTests (m_testCtx));
380 addChild(sparse::createTests (m_testCtx));
381 }
382
383 } // vkt
384