1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
4 *
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 */ /*!
21 * \file
22 * \brief CTS runner.
23 */ /*-------------------------------------------------------------------*/
24
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
31 #include "tcuApp.hpp"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
35
36 #include <iterator>
37
38 namespace glcts
39 {
40
41 using std::vector;
42 using std::string;
43
44 // RunSession
45
46 class RunSession
47 {
48 public:
RunSession(tcu::Platform & platform,tcu::Archive & archive,const int numArgs,const char * const * args)49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
50 : m_cmdLine(numArgs, args)
51 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
52 , m_app(platform, archive, m_log, m_cmdLine)
53 {
54 }
55
iterate(void)56 inline bool iterate(void)
57 {
58 return m_app.iterate();
59 }
60
getResult(void) const61 inline const tcu::TestRunStatus& getResult(void) const
62 {
63 return m_app.getResult();
64 }
65
66 private:
67 tcu::CommandLine m_cmdLine;
68 tcu::TestLog m_log;
69 tcu::App m_app;
70 };
71
appendConfigArgs(const Config & config,std::vector<std::string> & args,const char * fboConfig)72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
73 {
74 if (fboConfig != NULL)
75 {
76 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
77 args.push_back("--deqp-surface-type=fbo");
78 }
79
80 if (config.type != CONFIGTYPE_DEFAULT)
81 {
82 // \todo [2013-05-06 pyry] Test all surface types for some configs?
83 if (fboConfig == NULL)
84 {
85 if (config.surfaceTypes & SURFACETYPE_WINDOW)
86 args.push_back("--deqp-surface-type=window");
87 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
88 args.push_back("--deqp-surface-type=pbuffer");
89 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
90 args.push_back("--deqp-surface-type=pixmap");
91 }
92
93 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
94
95 if (config.type == CONFIGTYPE_EGL)
96 args.push_back("--deqp-gl-context-type=egl");
97 else if (config.type == CONFIGTYPE_WGL)
98 args.push_back("--deqp-gl-context-type=wgl");
99 }
100 }
101
102 typedef struct configInfo
103 {
104 deInt32 redBits;
105 deInt32 greenBits;
106 deInt32 blueBits;
107 deInt32 alphaBits;
108 deInt32 depthBits;
109 deInt32 stencilBits;
110 deInt32 samples;
111 } configInfo;
112
parseConfigBitsFromName(const char * configName)113 static configInfo parseConfigBitsFromName(const char* configName)
114 {
115 configInfo cfgInfo;
116 static const struct
117 {
118 const char* name;
119 int redBits;
120 int greenBits;
121 int blueBits;
122 int alphaBits;
123 } colorCfgs[] = {
124 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
125 };
126 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
127 {
128 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
129 {
130 cfgInfo.redBits = colorCfgs[ndx].redBits;
131 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
132 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
133 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
134
135 configName += strlen(colorCfgs[ndx].name);
136 break;
137 }
138 }
139
140 static const struct
141 {
142 const char* name;
143 int depthBits;
144 } depthCfgs[] = {
145 { "d0", 0 }, { "d24", 24 },
146 };
147 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
148 {
149 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
150 {
151 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
152
153 configName += strlen(depthCfgs[ndx].name);
154 break;
155 }
156 }
157
158 static const struct
159 {
160 const char* name;
161 int stencilBits;
162 } stencilCfgs[] = {
163 { "s0", 0 }, { "s8", 8 },
164 };
165 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
166 {
167 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
168 {
169 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
170
171 configName += strlen(stencilCfgs[ndx].name);
172 break;
173 }
174 }
175
176 static const struct
177 {
178 const char* name;
179 int samples;
180 } multiSampleCfgs[] = {
181 { "ms0", 0 }, { "ms4", 4 },
182 };
183 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
184 {
185 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
186 {
187 cfgInfo.samples = multiSampleCfgs[ndx].samples;
188
189 configName += strlen(multiSampleCfgs[ndx].name);
190 break;
191 }
192 }
193
194 return cfgInfo;
195 }
196
getApiName(glu::ApiType apiType)197 static const char* getApiName(glu::ApiType apiType)
198 {
199 if (apiType == glu::ApiType::es(2, 0))
200 return "gles2";
201 else if (apiType == glu::ApiType::es(3, 0))
202 return "gles3";
203 else if (apiType == glu::ApiType::es(3, 1))
204 return "gles31";
205 else if (apiType == glu::ApiType::es(3, 2))
206 return "gles32";
207 else if (apiType == glu::ApiType::core(3, 0))
208 return "gl30";
209 else if (apiType == glu::ApiType::core(3, 1))
210 return "gl31";
211 else if (apiType == glu::ApiType::core(3, 2))
212 return "gl32";
213 else if (apiType == glu::ApiType::core(3, 3))
214 return "gl33";
215 else if (apiType == glu::ApiType::core(4, 0))
216 return "gl40";
217 else if (apiType == glu::ApiType::core(4, 1))
218 return "gl41";
219 else if (apiType == glu::ApiType::core(4, 2))
220 return "gl42";
221 else if (apiType == glu::ApiType::core(4, 3))
222 return "gl43";
223 else if (apiType == glu::ApiType::core(4, 4))
224 return "gl44";
225 else if (apiType == glu::ApiType::core(4, 5))
226 return "gl45";
227 else if (apiType == glu::ApiType::core(4, 6))
228 return "gl46";
229 else
230 throw std::runtime_error("Unknown context type");
231 }
232
getCaseListFileOption(const char * mustpassDir,const char * apiName,const char * mustpassName)233 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)
234 {
235 #if DE_OS == DE_OS_ANDROID
236 const string case_list_option = "--deqp-caselist-resource=";
237 #else
238 const string case_list_option = "--deqp-caselist-file=";
239 #endif
240 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
241 }
242
getLogFileName(const char * apiName,const char * configName,const int iterId,const int runId,const int width,const int height,const int seed)243 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId,
244 const int width, const int height, const int seed)
245 {
246 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
247 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
248 if (seed != -1)
249 {
250 res += "-seed-" + de::toString(seed);
251 }
252 res += ".qpa";
253
254 return res;
255 }
256
getBaseOptions(std::vector<std::string> & args,const char * mustpassDir,const char * apiName,const char * configName,const char * screenRotation,int width,int height)257 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName,
258 const char* configName, const char* screenRotation, int width, int height)
259 {
260 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
261 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
262 args.push_back(string("--deqp-surface-width=") + de::toString(width));
263 args.push_back(string("--deqp-surface-height=") + de::toString(height));
264 args.push_back("--deqp-watchdog=disable");
265 }
266
isGLConfigCompatible(configInfo cfgInfo,const AOSPConfig & config)267 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
268 {
269 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
270 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
271 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
272 cfgInfo.samples == config.samples;
273 }
274
getTestRunsForAOSPEGL(vector<TestRunParams> & runs,const ConfigList & configs)275 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
276 {
277 #include "glcAospMustpassEgl.hpp"
278
279 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
280 {
281 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
282
283 vector<AOSPConfig>::const_iterator cfgIter;
284 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
285 {
286 // find first compatible config
287 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
288 {
289 break;
290 }
291 }
292
293 if (cfgIter == configs.aospConfigs.end())
294 {
295 // No suitable configuration found. Skipping EGL tests
296 continue;
297 }
298
299 const char* apiName = "egl";
300
301 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
302 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
303
304 TestRunParams params;
305 params.logFilename =
306 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
307 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
308 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
309
310 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
311
312 runs.push_back(params);
313 }
314 }
315
getTestRunsForAOSPES(vector<TestRunParams> & runs,const ConfigList & configs,const glu::ApiType apiType)316 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
317 {
318 #include "glcAospMustpassEs.hpp"
319
320 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
321 {
322 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
323 continue;
324
325 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
326
327 vector<AOSPConfig>::const_iterator cfgIter;
328 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
329 {
330 // find first compatible config
331 if (isGLConfigCompatible(cfgInfo, *cfgIter))
332 {
333 break;
334 }
335 }
336
337 if (cfgIter == configs.aospConfigs.end())
338 {
339 TCU_FAIL(("No suitable configuration found for GL config " +
340 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
341 .c_str());
342 return;
343 }
344
345 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
346
347 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
348 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
349
350 TestRunParams params;
351 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
352 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
353 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
354
355 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
356
357 //set surface type
358 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
359 params.args.push_back("--deqp-surface-type=window");
360 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
361 params.args.push_back("--deqp-surface-type=pbuffer");
362 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
363 params.args.push_back("--deqp-surface-type=pixmap");
364 runs.push_back(params);
365 }
366 }
367
getTestRunsForNoContext(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)368 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
369 const int numRunParams, const char* mustpassDir)
370 {
371 vector<Config>::const_iterator cfgIter = configs.configs.begin();
372
373 for (int i = 0; i < numRunParams; ++i)
374 {
375 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
376 continue;
377
378 const char* apiName = getApiName(runParams[i].apiType);
379
380 const int width = runParams[i].surfaceWidth;
381 const int height = runParams[i].surfaceHeight;
382 const int seed = runParams[i].baseSeed;
383
384 TestRunParams params;
385 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
386
387 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
388 height);
389
390 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
391
392 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
393
394 runs.push_back(params);
395 }
396 }
397
getTestRunsForNoContextES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)398 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
399 {
400 #include "glcKhronosMustpassEsNocontext.hpp"
401 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
402 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
403 }
404
getTestRunsForES(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)405 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
406 {
407 getTestRunsForAOSPEGL(runs, configs);
408 getTestRunsForAOSPES(runs, configs, type);
409 getTestRunsForNoContextES(type, runs, configs);
410
411 #include "glcKhronosMustpassEs.hpp"
412
413 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
414 {
415 const bool isFirst = cfgIter == configs.configs.begin();
416 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
417 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
418 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
419
420 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
421 {
422 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
423 continue;
424
425 const char* apiName = getApiName(runParams[runNdx].apiType);
426
427 const int width = runParams[runNdx].surfaceWidth;
428 const int height = runParams[runNdx].surfaceHeight;
429 const int seed = runParams[runNdx].baseSeed;
430
431 TestRunParams params;
432
433 params.logFilename =
434 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
435
436 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
437 runParams[runNdx].screenRotation, width, height);
438 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
439
440 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
441
442 runs.push_back(params);
443 }
444 }
445 }
446
getTestRunsForNoContextGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)447 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
448 {
449 #include "glcKhronosMustpassGlNocontext.hpp"
450 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
451 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
452 }
453
getTestRunsForGL(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)454 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
455 {
456 getTestRunsForNoContextGL(type, runs, configs);
457 #include "glcKhronosMustpassGl.hpp"
458
459 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
460 {
461 const bool isFirst = cfgIter == configs.configs.begin();
462 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
463 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
464 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
465
466 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
467 {
468 if (type != runParams[runNdx].apiType)
469 continue;
470
471 const char* apiName = getApiName(runParams[runNdx].apiType);
472
473 const int width = runParams[runNdx].surfaceWidth;
474 const int height = runParams[runNdx].surfaceHeight;
475 const int seed = runParams[runNdx].baseSeed;
476
477 TestRunParams params;
478
479 params.logFilename =
480 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
481
482 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
483 runParams[runNdx].screenRotation, width, height);
484 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
485
486 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
487
488 runs.push_back(params);
489 }
490 }
491 }
492
getTestRunParams(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)493 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
494 {
495 switch (type.getProfile())
496 {
497 case glu::PROFILE_CORE:
498 getTestRunsForGL(type, configs, runs);
499 break;
500 case glu::PROFILE_ES:
501 getTestRunsForES(type, configs, runs);
502 break;
503 default:
504 throw std::runtime_error("Unknown context type");
505 }
506 }
507
508 struct FileDeleter
509 {
operator ()glcts::FileDeleter510 void operator()(FILE* file) const
511 {
512 if (file)
513 fclose(file);
514 }
515 };
516
517 struct XmlWriterDeleter
518 {
operator ()glcts::XmlWriterDeleter519 void operator()(qpXmlWriter* writer) const
520 {
521 if (writer)
522 qpXmlWriter_destroy(writer);
523 }
524 };
525
getRunTypeName(glu::ApiType type)526 static const char* getRunTypeName(glu::ApiType type)
527 {
528 if (type == glu::ApiType::es(2, 0))
529 return "es2";
530 else if (type == glu::ApiType::es(3, 0))
531 return "es3";
532 else if (type == glu::ApiType::es(3, 1))
533 return "es31";
534 else if (type == glu::ApiType::es(3, 2))
535 return "es32";
536 else if (type == glu::ApiType::core(3, 0))
537 return "gl30";
538 else if (type == glu::ApiType::core(3, 1))
539 return "gl31";
540 else if (type == glu::ApiType::core(3, 2))
541 return "gl32";
542 else if (type == glu::ApiType::core(3, 3))
543 return "gl33";
544 else if (type == glu::ApiType::core(4, 0))
545 return "gl40";
546 else if (type == glu::ApiType::core(4, 1))
547 return "gl41";
548 else if (type == glu::ApiType::core(4, 2))
549 return "gl42";
550 else if (type == glu::ApiType::core(4, 3))
551 return "gl43";
552 else if (type == glu::ApiType::core(4, 4))
553 return "gl44";
554 else if (type == glu::ApiType::core(4, 5))
555 return "gl45";
556 else if (type == glu::ApiType::core(4, 6))
557 return "gl46";
558 else
559 return DE_NULL;
560 }
561
562 #define XML_CHECK(X) \
563 if (!(X)) \
564 throw tcu::Exception("Writing XML failed")
565
writeRunSummary(const TestRunSummary & summary,const char * filename)566 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
567 {
568 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
569 if (!out)
570 throw tcu::Exception(string("Failed to open ") + filename);
571
572 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
573 if (!writer)
574 throw std::bad_alloc();
575
576 XML_CHECK(qpXmlWriter_startDocument(writer.get()));
577
578 {
579 qpXmlAttribute attribs[2];
580
581 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
582 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
583
584 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
585 }
586
587 // Config run
588 {
589 qpXmlAttribute attribs[1];
590 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
591 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
592 qpXmlWriter_endElement(writer.get(), "Configs"));
593 }
594
595 // Record test run parameters (log filename & command line).
596 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
597 ++runIter)
598 {
599 string cmdLine;
600 qpXmlAttribute attribs[2];
601
602 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
603 {
604 if (argIter != runIter->args.begin())
605 cmdLine += " ";
606 cmdLine += *argIter;
607 }
608
609 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
610 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
611
612 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
613 qpXmlWriter_endElement(writer.get(), "TestRun"));
614 }
615
616 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
617 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
618 }
619
620 #undef XML_CHECK
621
TestRunner(tcu::Platform & platform,tcu::Archive & archive,const char * logDirPath,glu::ApiType type,deUint32 flags)622 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
623 deUint32 flags)
624 : m_platform(platform)
625 , m_archive(archive)
626 , m_logDirPath(logDirPath)
627 , m_type(type)
628 , m_flags(flags)
629 , m_iterState(ITERATE_INIT)
630 , m_curSession(DE_NULL)
631 , m_sessionsExecuted(0)
632 , m_sessionsPassed(0)
633 , m_sessionsFailed(0)
634 {
635 }
636
~TestRunner(void)637 TestRunner::~TestRunner(void)
638 {
639 delete m_curSession;
640 }
641
iterate(void)642 bool TestRunner::iterate(void)
643 {
644 switch (m_iterState)
645 {
646 case ITERATE_INIT:
647 init();
648 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
649 return true;
650
651 case ITERATE_DEINIT:
652 deinit();
653 m_iterState = ITERATE_INIT;
654 return false;
655
656 case ITERATE_INIT_SESSION:
657 DE_ASSERT(m_sessionIter != m_runSessions.end());
658 initSession(*m_sessionIter);
659 if (m_flags & PRINT_SUMMARY)
660 m_iterState = ITERATE_DEINIT_SESSION;
661 else
662 m_iterState = ITERATE_ITERATE_SESSION;
663 return true;
664
665 case ITERATE_DEINIT_SESSION:
666 deinitSession();
667 ++m_sessionIter;
668 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
669 return true;
670
671 case ITERATE_ITERATE_SESSION:
672 if (!iterateSession())
673 m_iterState = ITERATE_DEINIT_SESSION;
674 return true;
675
676 default:
677 DE_ASSERT(false);
678 return false;
679 }
680 }
681
init(void)682 void TestRunner::init(void)
683 {
684 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
685
686 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
687
688 m_summary.runType = m_type;
689
690 // Get list of configs to test.
691 ConfigList configList;
692 getDefaultConfigList(m_platform, m_type, configList);
693
694 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
695 (int)configList.excludedConfigs.size());
696
697 // Config list run.
698 {
699 const char* configLogFilename = "configs.qpa";
700 TestRunParams configRun;
701
702 configRun.logFilename = configLogFilename;
703 configRun.args.push_back("--deqp-case=CTS-Configs.*");
704 m_runSessions.push_back(configRun);
705
706 m_summary.configLogFilename = configLogFilename;
707 }
708
709 // Conformance test type specific runs
710 getTestRunParams(m_type, configList, m_runSessions);
711
712 // Record run params for summary.
713 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
714 ++runIter)
715 m_summary.runParams.push_back(*runIter);
716
717 // Session iterator
718 m_sessionIter = m_runSessions.begin();
719 }
720
deinit(void)721 void TestRunner::deinit(void)
722 {
723 // Print out totals.
724 bool isConformant = m_sessionsExecuted == m_sessionsPassed;
725 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
726 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
727 isConformant ? "PASSED" : "FAILED");
728
729 m_summary.isConformant = isConformant;
730
731 // Write out summary.
732 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
733
734 m_runSessions.clear();
735 m_summary.clear();
736 }
737
initSession(const TestRunParams & runParams)738 void TestRunner::initSession(const TestRunParams& runParams)
739 {
740 DE_ASSERT(!m_curSession);
741
742 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
743
744 // Compute final args for run.
745 vector<string> args(runParams.args);
746 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
747
748 if (!(m_flags & VERBOSE_IMAGES))
749 args.push_back("--deqp-log-images=disable");
750
751 if (!(m_flags & VERBOSE_SHADERS))
752 args.push_back("--deqp-log-shader-sources=disable");
753
754 std::ostringstream ostr;
755 std::ostream_iterator<string> out_it(ostr, ", ");
756 std::copy(args.begin(), args.end(), out_it);
757 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
758
759 // Translate to argc, argv
760 vector<const char*> argv;
761 argv.push_back("cts-runner"); // Dummy binary name
762 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
763 argv.push_back(i->c_str());
764
765 // Create session
766 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
767 }
768
deinitSession(void)769 void TestRunner::deinitSession(void)
770 {
771 DE_ASSERT(m_curSession);
772
773 // Collect results.
774 // \note NotSupported is treated as pass.
775 const tcu::TestRunStatus& result = m_curSession->getResult();
776 bool isOk =
777 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
778
779 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
780
781 m_sessionsExecuted += 1;
782 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
783
784 delete m_curSession;
785 m_curSession = DE_NULL;
786 }
787
iterateSession(void)788 inline bool TestRunner::iterateSession(void)
789 {
790 return m_curSession->iterate();
791 }
792
793 } // glcts
794