1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Command line test executor.
22 *//*--------------------------------------------------------------------*/
23
24 #include "xeBatchExecutor.hpp"
25 #include "xeTestCaseListParser.hpp"
26 #include "xeTcpIpLink.hpp"
27 #include "xeLocalTcpIpLink.hpp"
28 #include "xeTestResultParser.hpp"
29 #include "xeTestLogWriter.hpp"
30 #include "deDirectoryIterator.hpp"
31 #include "deCommandLine.hpp"
32 #include "deString.h"
33
34 #include <vector>
35 #include <string>
36 #include <cstdio>
37 #include <cstdlib>
38 #include <fstream>
39 #include <memory>
40 #include <algorithm>
41 #include <iostream>
42 #include <sstream>
43
44 // Command line arguments.
45 namespace opt
46 {
47
48 DE_DECLARE_COMMAND_LINE_OPT(StartServer, std::string);
49 DE_DECLARE_COMMAND_LINE_OPT(Host, std::string);
50 DE_DECLARE_COMMAND_LINE_OPT(Port, int);
51 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, std::string);
52 DE_DECLARE_COMMAND_LINE_OPT(TestSet, std::vector<std::string>);
53 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, std::vector<std::string>);
54 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, std::string);
55 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, std::string);
56 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, std::string);
57 DE_DECLARE_COMMAND_LINE_OPT(Summary, bool);
58
59 // TargetConfiguration
60 DE_DECLARE_COMMAND_LINE_OPT(BinaryName, std::string);
61 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, std::string);
62 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, std::string);
63
parseCommaSeparatedList(const char * src,std::vector<std::string> * dst)64 static void parseCommaSeparatedList (const char* src, std::vector<std::string>* dst)
65 {
66 std::istringstream inStr (src);
67 std::string comp;
68
69 while (std::getline(inStr, comp, ','))
70 dst->push_back(comp);
71 }
72
registerOptions(de::cmdline::Parser & parser)73 void registerOptions (de::cmdline::Parser& parser)
74 {
75 using de::cmdline::Option;
76 using de::cmdline::NamedValue;
77
78 static const NamedValue<bool> s_yesNo[] =
79 {
80 { "yes", true },
81 { "no", false }
82 };
83
84 parser << Option<StartServer> ("s", "start-server", "Start local execserver", "")
85 << Option<Host> ("c", "connect", "Connect to host", "127.0.0.1")
86 << Option<Port> ("p", "port", "Select TCP port to use", "50016")
87 << Option<CaseListDir> ("cd", "caselistdir", "Path to test case XML files", ".")
88 << Option<TestSet> ("t", "testset", "Test set", parseCommaSeparatedList, "")
89 << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters", parseCommaSeparatedList, "")
90 << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log", "")
91 << Option<TestLogFile> ("o", "out", "Output test log filename", "")
92 << Option<InfoLogFile> ("i", "info", "Output info log filename", "")
93 << Option<Summary> (DE_NULL, "summary", "Print summary at the end", s_yesNo, "yes")
94 << Option<BinaryName> ("b", "binaryname", "Test binary path, relative to working directory", "")
95 << Option<WorkingDir> ("wd", "workdir", "Working directory for test execution", "")
96 << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for test binary", "");
97 }
98
99 } // opt
100
101 using std::vector;
102 using std::string;
103
104 struct CommandLine
105 {
CommandLineCommandLine106 CommandLine (void)
107 : port (0)
108 , summary (false)
109 {
110 }
111
112 xe::TargetConfiguration targetCfg;
113 std::string serverBin;
114 std::string host;
115 int port;
116 std::string caseListDir;
117 std::vector<std::string> testset;
118 std::vector<std::string> exclude;
119 std::string inFile;
120 std::string outFile;
121 std::string infoFile;
122 bool summary;
123 };
124
parseCommandLine(CommandLine & cmdLine,int argc,const char * const * argv)125 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
126 {
127 de::cmdline::Parser parser;
128 de::cmdline::CommandLine opts;
129
130 XE_CHECK(argc >= 1);
131
132 opt::registerOptions(parser);
133
134 if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
135 {
136 std::cout << argv[0] << " [options]\n";
137 parser.help(std::cout);
138 return false;
139 }
140
141 cmdLine.serverBin = opts.getOption<opt::StartServer>();
142 cmdLine.host = opts.getOption<opt::Host>();
143 cmdLine.port = opts.getOption<opt::Port>();
144 cmdLine.caseListDir = opts.getOption<opt::CaseListDir>();
145 cmdLine.testset = opts.getOption<opt::TestSet>();
146 cmdLine.exclude = opts.getOption<opt::ExcludeSet>();
147 cmdLine.inFile = opts.getOption<opt::ContinueFile>();
148 cmdLine.outFile = opts.getOption<opt::TestLogFile>();
149 cmdLine.infoFile = opts.getOption<opt::InfoLogFile>();
150 cmdLine.summary = opts.getOption<opt::Summary>();
151 cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>();
152 cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>();
153 cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>();
154
155 return true;
156 }
157
checkCasePathPatternMatch(const char * pattern,const char * casePath,bool isTestGroup)158 static bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
159 {
160 int ptrnPos = 0;
161 int casePos = 0;
162
163 for (;;)
164 {
165 char c = casePath[casePos];
166 char p = pattern[ptrnPos];
167
168 if (p == '*')
169 {
170 /* Recurse to rest of positions. */
171 int next = casePos;
172 for (;;)
173 {
174 if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
175 return DE_TRUE;
176
177 if (casePath[next] == 0)
178 return DE_FALSE; /* No match found. */
179 else
180 next += 1;
181 }
182 DE_ASSERT(DE_FALSE);
183 }
184 else if (c == 0 && p == 0)
185 return true;
186 else if (c == 0)
187 {
188 /* Incomplete match is ok for test groups. */
189 return isTestGroup;
190 }
191 else if (c != p)
192 return false;
193
194 casePos += 1;
195 ptrnPos += 1;
196 }
197
198 DE_ASSERT(false);
199 return false;
200 }
201
readCaseList(xe::TestGroup * root,const char * filename)202 static void readCaseList (xe::TestGroup* root, const char* filename)
203 {
204 xe::TestCaseListParser caseListParser;
205 std::ifstream in (filename, std::ios_base::binary);
206 deUint8 buf[1024];
207
208 XE_CHECK(in.good());
209
210 caseListParser.init(root);
211
212 for (;;)
213 {
214 in.read((char*)&buf[0], sizeof(buf));
215 int numRead = (int)in.gcount();
216
217 if (numRead > 0)
218 caseListParser.parse(&buf[0], numRead);
219
220 if (numRead < (int)sizeof(buf))
221 break; // EOF
222 }
223 }
224
readCaseLists(xe::TestRoot & root,const char * caseListDir)225 static void readCaseLists (xe::TestRoot& root, const char* caseListDir)
226 {
227 de::DirectoryIterator iter(caseListDir);
228
229 for (; iter.hasItem(); iter.next())
230 {
231 de::FilePath item = iter.getItem();
232
233 if (item.getType() == de::FilePath::TYPE_FILE)
234 {
235 std::string baseName = item.getBaseName();
236 if (baseName.find("-cases.xml") == baseName.length()-10)
237 {
238 std::string packageName = baseName.substr(0, baseName.length()-10);
239 xe::TestGroup* package = root.createGroup(packageName.c_str(), "");
240
241 readCaseList(package, item.getPath());
242 }
243 }
244 }
245 }
246
addMatchingCases(const xe::TestGroup & group,xe::TestSet & testSet,const char * filter)247 static void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
248 {
249 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
250 {
251 const xe::TestNode* child = group.getChild(childNdx);
252 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
253 const std::string fullPath = child->getFullPath();
254
255 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
256 {
257 if (isGroup)
258 {
259 // Recurse into group.
260 addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
261 }
262 else
263 {
264 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
265 testSet.add(child);
266 }
267 }
268 }
269 }
270
removeMatchingCases(const xe::TestGroup & group,xe::TestSet & testSet,const char * filter)271 static void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
272 {
273 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
274 {
275 const xe::TestNode* child = group.getChild(childNdx);
276 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
277 const std::string fullPath = child->getFullPath();
278
279 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
280 {
281 if (isGroup)
282 {
283 // Recurse into group.
284 removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
285 }
286 else
287 {
288 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
289 testSet.remove(child);
290 }
291 }
292 }
293 }
294
295 class BatchResultHandler : public xe::TestLogHandler
296 {
297 public:
BatchResultHandler(xe::BatchResult * batchResult)298 BatchResultHandler (xe::BatchResult* batchResult)
299 : m_batchResult(batchResult)
300 {
301 }
302
setSessionInfo(const xe::SessionInfo & sessionInfo)303 void setSessionInfo (const xe::SessionInfo& sessionInfo)
304 {
305 m_batchResult->getSessionInfo() = sessionInfo;
306 }
307
startTestCaseResult(const char * casePath)308 xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
309 {
310 // \todo [2012-11-01 pyry] What to do with duplicate results?
311 if (m_batchResult->hasTestCaseResult(casePath))
312 return m_batchResult->getTestCaseResult(casePath);
313 else
314 return m_batchResult->createTestCaseResult(casePath);
315 }
316
testCaseResultUpdated(const xe::TestCaseResultPtr &)317 void testCaseResultUpdated (const xe::TestCaseResultPtr&)
318 {
319 }
320
testCaseResultComplete(const xe::TestCaseResultPtr &)321 void testCaseResultComplete (const xe::TestCaseResultPtr&)
322 {
323 }
324
325 private:
326 xe::BatchResult* m_batchResult;
327 };
328
readLogFile(xe::BatchResult * batchResult,const char * filename)329 static void readLogFile (xe::BatchResult* batchResult, const char* filename)
330 {
331 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
332 BatchResultHandler handler (batchResult);
333 xe::TestLogParser parser (&handler);
334 deUint8 buf [1024];
335 int numRead = 0;
336
337 for (;;)
338 {
339 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
340 numRead = (int)in.gcount();
341
342 if (numRead <= 0)
343 break;
344
345 parser.parse(&buf[0], numRead);
346 }
347
348 in.close();
349 }
350
printBatchResultSummary(const xe::TestNode * root,const xe::TestSet & testSet,const xe::BatchResult & batchResult)351 static void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
352 {
353 int countByStatusCode[xe::TESTSTATUSCODE_LAST];
354 std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
355
356 for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
357 {
358 const xe::TestNode* node = *iter;
359 if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
360 {
361 const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node);
362 std::string fullPath;
363 xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING;
364 testCase->getFullPath(fullPath);
365
366 // Parse result data if such exists.
367 if (batchResult.hasTestCaseResult(fullPath.c_str()))
368 {
369 xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str());
370 xe::TestCaseResult result;
371 xe::TestResultParser parser;
372
373 xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
374 statusCode = result.statusCode;
375 }
376
377 countByStatusCode[statusCode] += 1;
378 }
379 }
380
381 printf("\nTest run summary:\n");
382 int totalCases = 0;
383 for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
384 {
385 if (countByStatusCode[code] > 0)
386 printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
387
388 totalCases += countByStatusCode[code];
389 }
390 printf(" %20s: %5d\n", "Total", totalCases);
391 }
392
writeInfoLog(const xe::InfoLog & log,const char * filename)393 static void writeInfoLog (const xe::InfoLog& log, const char* filename)
394 {
395 std::ofstream out(filename, std::ios_base::binary);
396 XE_CHECK(out.good());
397 out.write((const char*)log.getBytes(), log.getSize());
398 out.close();
399 }
400
createCommLink(const CommandLine & cmdLine)401 static xe::CommLink* createCommLink (const CommandLine& cmdLine)
402 {
403 if (!cmdLine.serverBin.empty())
404 {
405 xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
406 try
407 {
408 link->start(cmdLine.serverBin.c_str(), DE_NULL, cmdLine.port);
409 return link;
410 }
411 catch (...)
412 {
413 delete link;
414 throw;
415 }
416 }
417 else
418 {
419 de::SocketAddress address;
420 address.setFamily(DE_SOCKETFAMILY_INET4);
421 address.setProtocol(DE_SOCKETPROTOCOL_TCP);
422 address.setHost(cmdLine.host.c_str());
423 address.setPort(cmdLine.port);
424
425 xe::TcpIpLink* link = new xe::TcpIpLink();
426 try
427 {
428 link->connect(address);
429 return link;
430 }
431 catch (...)
432 {
433 delete link;
434 throw;
435 }
436 }
437 }
438
runExecutor(const CommandLine & cmdLine)439 static void runExecutor (const CommandLine& cmdLine)
440 {
441 xe::TestRoot root;
442
443 // Read case list definitions.
444 readCaseLists(root, cmdLine.caseListDir.c_str());
445
446 // Build test set.
447 xe::TestSet testSet;
448
449 // Build test set.
450 for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
451 addMatchingCases(root, testSet, filterIter->c_str());
452
453 // Remove excluded cases.
454 for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
455 removeMatchingCases(root, testSet, filterIter->c_str());
456
457 // Initialize batch result.
458 xe::BatchResult batchResult;
459 xe::InfoLog infoLog;
460
461 // Read existing results from input file (if supplied).
462 if (!cmdLine.inFile.empty())
463 readLogFile(&batchResult, cmdLine.inFile.c_str());
464
465 // Initialize commLink.
466 std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine));
467
468 xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
469 executor.run();
470
471 commLink.reset();
472
473 if (!cmdLine.outFile.empty())
474 {
475 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
476 printf("Test log written to %s\n", cmdLine.outFile.c_str());
477 }
478
479 if (!cmdLine.infoFile.empty())
480 {
481 writeInfoLog(infoLog, cmdLine.infoFile.c_str());
482 printf("Info log written to %s\n", cmdLine.infoFile.c_str());
483 }
484
485 if (cmdLine.summary)
486 printBatchResultSummary(&root, testSet, batchResult);
487 }
488
main(int argc,const char * const * argv)489 int main (int argc, const char* const* argv)
490 {
491 CommandLine cmdLine;
492
493 if (!parseCommandLine(cmdLine, argc, argv))
494 return -1;
495
496 try
497 {
498 runExecutor(cmdLine);
499 }
500 catch (const std::exception& e)
501 {
502 printf("%s\n", e.what());
503 return -1;
504 }
505
506 return 0;
507 }
508