1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Command line test executor.
22 *//*--------------------------------------------------------------------*/
23
24 #include "xeBatchExecutor.hpp"
25 #include "xeLocalTcpIpLink.hpp"
26 #include "xeTcpIpLink.hpp"
27 #include "xeTestCaseListParser.hpp"
28 #include "xeTestLogWriter.hpp"
29 #include "xeTestResultParser.hpp"
30
31 #include "deCommandLine.hpp"
32 #include "deDirectoryIterator.hpp"
33 #include "deStringUtil.hpp"
34
35 #include "deString.h"
36
37 #include <algorithm>
38 #include <cstdio>
39 #include <cstdlib>
40 #include <fstream>
41 #include <iostream>
42 #include <memory>
43 #include <sstream>
44 #include <string>
45 #include <vector>
46
47 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32)
48 # include <signal.h>
49 #endif
50
51 using std::vector;
52 using std::string;
53
54 namespace
55 {
56
57 // Command line arguments.
58 namespace opt
59 {
60
61 DE_DECLARE_COMMAND_LINE_OPT(StartServer, string);
62 DE_DECLARE_COMMAND_LINE_OPT(Host, string);
63 DE_DECLARE_COMMAND_LINE_OPT(Port, int);
64 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string);
65 DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>);
66 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>);
67 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string);
68 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string);
69 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string);
70 DE_DECLARE_COMMAND_LINE_OPT(Summary, bool);
71
72 // TargetConfiguration
73 DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string);
74 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string);
75 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string);
76
parseCommaSeparatedList(const char * src,vector<string> * dst)77 void parseCommaSeparatedList (const char* src, vector<string>* dst)
78 {
79 std::istringstream inStr (src);
80 string comp;
81
82 while (std::getline(inStr, comp, ','))
83 dst->push_back(comp);
84 }
85
registerOptions(de::cmdline::Parser & parser)86 void registerOptions (de::cmdline::Parser& parser)
87 {
88 using de::cmdline::Option;
89 using de::cmdline::NamedValue;
90
91 static const NamedValue<bool> s_yesNo[] =
92 {
93 { "yes", true },
94 { "no", false }
95 };
96
97 parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.")
98 << Option<Host> ("c", "connect", "Connect to host. Address of the execserver.")
99 << Option<Port> ("p", "port", "TCP port of the execserver.", "50016")
100 << Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".")
101 << Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList)
102 << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "")
103 << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.")
104 << Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa")
105 << Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt")
106 << Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes")
107 << Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>")
108 << Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".")
109 << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", "");
110 }
111
112 } // opt
113
114 enum RunMode
115 {
116 RUNMODE_CONNECT,
117 RUNMODE_START_SERVER
118 };
119
120 struct CommandLine
121 {
CommandLine__anon5b82974c0111::CommandLine122 CommandLine (void)
123 : port (0)
124 , summary (false)
125 {
126 }
127
128 xe::TargetConfiguration targetCfg;
129 RunMode runMode;
130 string serverBinOrAddress;
131 int port;
132 string caseListDir;
133 vector<string> testset;
134 vector<string> exclude;
135 string inFile;
136 string outFile;
137 string infoFile;
138 bool summary;
139 };
140
parseCommandLine(CommandLine & cmdLine,int argc,const char * const * argv)141 bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
142 {
143 de::cmdline::Parser parser;
144 de::cmdline::CommandLine opts;
145
146 XE_CHECK(argc >= 1);
147
148 opt::registerOptions(parser);
149
150 if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
151 {
152 std::cout << argv[0] << " [options]\n";
153 parser.help(std::cout);
154 return false;
155 }
156
157 if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>())
158 {
159 std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl;
160 return false;
161 }
162 else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>())
163 {
164 std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl;
165 return false;
166 }
167
168 if (!opts.hasOption<opt::TestSet>())
169 {
170 std::cout << "Invalid command line arguments. --testset not defined." << std::endl;
171 return false;
172 }
173
174 if (opts.hasOption<opt::StartServer>())
175 {
176 cmdLine.runMode = RUNMODE_START_SERVER;
177 cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>();
178 }
179 else
180 {
181 cmdLine.runMode = RUNMODE_CONNECT;
182 cmdLine.serverBinOrAddress = opts.getOption<opt::Host>();
183 }
184
185 if (opts.hasOption<opt::ContinueFile>())
186 {
187 cmdLine.inFile = opts.getOption<opt::ContinueFile>();
188
189 if (cmdLine.inFile.empty())
190 {
191 std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl;
192 return false;
193 }
194 }
195
196 cmdLine.port = opts.getOption<opt::Port>();
197 cmdLine.caseListDir = opts.getOption<opt::CaseListDir>();
198 cmdLine.testset = opts.getOption<opt::TestSet>();
199 cmdLine.exclude = opts.getOption<opt::ExcludeSet>();
200 cmdLine.outFile = opts.getOption<opt::TestLogFile>();
201 cmdLine.infoFile = opts.getOption<opt::InfoLogFile>();
202 cmdLine.summary = opts.getOption<opt::Summary>();
203 cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>();
204 cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>();
205 cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>();
206
207 return true;
208 }
209
checkCasePathPatternMatch(const char * pattern,const char * casePath,bool isTestGroup)210 bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
211 {
212 int ptrnPos = 0;
213 int casePos = 0;
214
215 for (;;)
216 {
217 char c = casePath[casePos];
218 char p = pattern[ptrnPos];
219
220 if (p == '*')
221 {
222 /* Recurse to rest of positions. */
223 int next = casePos;
224 for (;;)
225 {
226 if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
227 return DE_TRUE;
228
229 if (casePath[next] == 0)
230 return DE_FALSE; /* No match found. */
231 else
232 next += 1;
233 }
234 DE_ASSERT(DE_FALSE);
235 }
236 else if (c == 0 && p == 0)
237 return true;
238 else if (c == 0)
239 {
240 /* Incomplete match is ok for test groups. */
241 return isTestGroup;
242 }
243 else if (c != p)
244 return false;
245
246 casePos += 1;
247 ptrnPos += 1;
248 }
249
250 DE_ASSERT(false);
251 return false;
252 }
253
readCaseList(xe::TestGroup * root,const char * filename)254 void readCaseList (xe::TestGroup* root, const char* filename)
255 {
256 xe::TestCaseListParser caseListParser;
257 std::ifstream in (filename, std::ios_base::binary);
258 deUint8 buf[1024];
259
260 XE_CHECK(in.good());
261
262 caseListParser.init(root);
263
264 for (;;)
265 {
266 in.read((char*)&buf[0], sizeof(buf));
267 int numRead = (int)in.gcount();
268
269 if (numRead > 0)
270 caseListParser.parse(&buf[0], numRead);
271
272 if (numRead < (int)sizeof(buf))
273 break; // EOF
274 }
275 }
276
readCaseLists(xe::TestRoot & root,const char * caseListDir)277 void readCaseLists (xe::TestRoot& root, const char* caseListDir)
278 {
279 int testCaseListCount = 0;
280 de::DirectoryIterator iter (caseListDir);
281
282 for (; iter.hasItem(); iter.next())
283 {
284 de::FilePath item = iter.getItem();
285
286 if (item.getType() == de::FilePath::TYPE_FILE)
287 {
288 string baseName = item.getBaseName();
289 if (baseName.find("-cases.xml") == baseName.length()-10)
290 {
291 string packageName = baseName.substr(0, baseName.length()-10);
292 xe::TestGroup* package = root.createGroup(packageName.c_str(), "");
293
294 readCaseList(package, item.getPath());
295 testCaseListCount++;
296 }
297 }
298 }
299
300 if (testCaseListCount == 0)
301 throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'");
302 }
303
addMatchingCases(const xe::TestGroup & group,xe::TestSet & testSet,const char * filter)304 void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
305 {
306 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
307 {
308 const xe::TestNode* child = group.getChild(childNdx);
309 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
310 const string fullPath = child->getFullPath();
311
312 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
313 {
314 if (isGroup)
315 {
316 // Recurse into group.
317 addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
318 }
319 else
320 {
321 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
322 testSet.add(child);
323 }
324 }
325 }
326 }
327
removeMatchingCases(const xe::TestGroup & group,xe::TestSet & testSet,const char * filter)328 void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
329 {
330 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
331 {
332 const xe::TestNode* child = group.getChild(childNdx);
333 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
334 const string fullPath = child->getFullPath();
335
336 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
337 {
338 if (isGroup)
339 {
340 // Recurse into group.
341 removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
342 }
343 else
344 {
345 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
346 testSet.remove(child);
347 }
348 }
349 }
350 }
351
352 class BatchResultHandler : public xe::TestLogHandler
353 {
354 public:
BatchResultHandler(xe::BatchResult * batchResult)355 BatchResultHandler (xe::BatchResult* batchResult)
356 : m_batchResult(batchResult)
357 {
358 }
359
setSessionInfo(const xe::SessionInfo & sessionInfo)360 void setSessionInfo (const xe::SessionInfo& sessionInfo)
361 {
362 m_batchResult->getSessionInfo() = sessionInfo;
363 }
364
startTestCaseResult(const char * casePath)365 xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
366 {
367 // \todo [2012-11-01 pyry] What to do with duplicate results?
368 if (m_batchResult->hasTestCaseResult(casePath))
369 return m_batchResult->getTestCaseResult(casePath);
370 else
371 return m_batchResult->createTestCaseResult(casePath);
372 }
373
testCaseResultUpdated(const xe::TestCaseResultPtr &)374 void testCaseResultUpdated (const xe::TestCaseResultPtr&)
375 {
376 }
377
testCaseResultComplete(const xe::TestCaseResultPtr &)378 void testCaseResultComplete (const xe::TestCaseResultPtr&)
379 {
380 }
381
382 private:
383 xe::BatchResult* m_batchResult;
384 };
385
readLogFile(xe::BatchResult * batchResult,const char * filename)386 void readLogFile (xe::BatchResult* batchResult, const char* filename)
387 {
388 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
389 BatchResultHandler handler (batchResult);
390 xe::TestLogParser parser (&handler);
391 deUint8 buf [1024];
392 int numRead = 0;
393
394 for (;;)
395 {
396 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
397 numRead = (int)in.gcount();
398
399 if (numRead <= 0)
400 break;
401
402 parser.parse(&buf[0], numRead);
403 }
404
405 in.close();
406 }
407
printBatchResultSummary(const xe::TestNode * root,const xe::TestSet & testSet,const xe::BatchResult & batchResult)408 void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
409 {
410 int countByStatusCode[xe::TESTSTATUSCODE_LAST];
411 std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
412
413 for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
414 {
415 const xe::TestNode* node = *iter;
416 if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
417 {
418 const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node);
419 string fullPath;
420 xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING;
421 testCase->getFullPath(fullPath);
422
423 // Parse result data if such exists.
424 if (batchResult.hasTestCaseResult(fullPath.c_str()))
425 {
426 xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str());
427 xe::TestCaseResult result;
428 xe::TestResultParser parser;
429
430 xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
431 statusCode = result.statusCode;
432 }
433
434 countByStatusCode[statusCode] += 1;
435 }
436 }
437
438 printf("\nTest run summary:\n");
439 int totalCases = 0;
440 for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
441 {
442 if (countByStatusCode[code] > 0)
443 printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
444
445 totalCases += countByStatusCode[code];
446 }
447 printf(" %20s: %5d\n", "Total", totalCases);
448 }
449
writeInfoLog(const xe::InfoLog & log,const char * filename)450 void writeInfoLog (const xe::InfoLog& log, const char* filename)
451 {
452 std::ofstream out(filename, std::ios_base::binary);
453 XE_CHECK(out.good());
454 out.write((const char*)log.getBytes(), log.getSize());
455 out.close();
456 }
457
createCommLink(const CommandLine & cmdLine)458 xe::CommLink* createCommLink (const CommandLine& cmdLine)
459 {
460 if (cmdLine.runMode == RUNMODE_START_SERVER)
461 {
462 xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
463 try
464 {
465 link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port);
466 return link;
467 }
468 catch (...)
469 {
470 delete link;
471 throw;
472 }
473 }
474 else if (cmdLine.runMode == RUNMODE_CONNECT)
475 {
476 de::SocketAddress address;
477
478 address.setFamily(DE_SOCKETFAMILY_INET4);
479 address.setProtocol(DE_SOCKETPROTOCOL_TCP);
480 address.setHost(cmdLine.serverBinOrAddress.c_str());
481 address.setPort(cmdLine.port);
482
483 xe::TcpIpLink* link = new xe::TcpIpLink();
484 try
485 {
486 std::string error;
487
488 link->connect(address);
489 return link;
490 }
491 catch (const std::exception& error)
492 {
493 delete link;
494 throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what());
495 }
496 catch (...)
497 {
498 delete link;
499 throw;
500 }
501 }
502 else
503 {
504 DE_ASSERT(false);
505 return DE_NULL;
506 }
507 }
508
509 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID)
510
511 static xe::BatchExecutor* s_executor = DE_NULL;
512
signalHandler(int,siginfo_t *,void *)513 void signalHandler (int, siginfo_t*, void*)
514 {
515 if (s_executor)
516 s_executor->cancel();
517 }
518
setupSignalHandler(xe::BatchExecutor * executor)519 void setupSignalHandler (xe::BatchExecutor* executor)
520 {
521 s_executor = executor;
522 struct sigaction sa;
523
524 sa.sa_sigaction = signalHandler;
525 sa.sa_flags = SA_SIGINFO | SA_RESTART;
526 sigfillset(&sa.sa_mask);
527
528 sigaction(SIGINT, &sa, DE_NULL);
529 }
530
resetSignalHandler(void)531 void resetSignalHandler (void)
532 {
533 struct sigaction sa;
534
535 sa.sa_handler = SIG_DFL;
536 sa.sa_flags = SA_RESTART;
537 sigfillset(&sa.sa_mask);
538
539 sigaction(SIGINT, &sa, DE_NULL);
540 s_executor = DE_NULL;
541 }
542
543 #elif (DE_OS == DE_OS_WIN32)
544
545 static xe::BatchExecutor* s_executor = DE_NULL;
546
signalHandler(int)547 void signalHandler (int)
548 {
549 if (s_executor)
550 s_executor->cancel();
551 }
552
setupSignalHandler(xe::BatchExecutor * executor)553 void setupSignalHandler (xe::BatchExecutor* executor)
554 {
555 s_executor = executor;
556 signal(SIGINT, signalHandler);
557 }
558
resetSignalHandler(void)559 void resetSignalHandler (void)
560 {
561 signal(SIGINT, SIG_DFL);
562 s_executor = DE_NULL;
563 }
564
565 #else
566
setupSignalHandler(xe::BatchExecutor *)567 void setupSignalHandler (xe::BatchExecutor*)
568 {
569 }
570
resetSignalHandler(void)571 void resetSignalHandler (void)
572 {
573 }
574
575 #endif
576
runExecutor(const CommandLine & cmdLine)577 void runExecutor (const CommandLine& cmdLine)
578 {
579 xe::TestRoot root;
580
581 // Read case list definitions.
582 readCaseLists(root, cmdLine.caseListDir.c_str());
583
584 // Build test set.
585 xe::TestSet testSet;
586
587 // Build test set.
588 for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
589 addMatchingCases(root, testSet, filterIter->c_str());
590
591 if (testSet.empty())
592 throw xe::Error("None of the test case lists contains tests matching any of the test sets.");
593
594 // Remove excluded cases.
595 for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
596 removeMatchingCases(root, testSet, filterIter->c_str());
597
598 // Initialize batch result.
599 xe::BatchResult batchResult;
600 xe::InfoLog infoLog;
601
602 // Read existing results from input file (if supplied).
603 if (!cmdLine.inFile.empty())
604 readLogFile(&batchResult, cmdLine.inFile.c_str());
605
606 // Initialize commLink.
607 std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine));
608
609 xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
610
611 try
612 {
613 setupSignalHandler(&executor);
614 executor.run();
615 resetSignalHandler();
616 }
617 catch (...)
618 {
619 resetSignalHandler();
620
621 if (!cmdLine.outFile.empty())
622 {
623 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
624 printf("Test log written to %s\n", cmdLine.outFile.c_str());
625 }
626
627 if (!cmdLine.infoFile.empty())
628 {
629 writeInfoLog(infoLog, cmdLine.infoFile.c_str());
630 printf("Info log written to %s\n", cmdLine.infoFile.c_str());
631 }
632
633 if (cmdLine.summary)
634 printBatchResultSummary(&root, testSet, batchResult);
635
636 throw;
637 }
638
639 if (!cmdLine.outFile.empty())
640 {
641 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
642 printf("Test log written to %s\n", cmdLine.outFile.c_str());
643 }
644
645 if (!cmdLine.infoFile.empty())
646 {
647 writeInfoLog(infoLog, cmdLine.infoFile.c_str());
648 printf("Info log written to %s\n", cmdLine.infoFile.c_str());
649 }
650
651 if (cmdLine.summary)
652 printBatchResultSummary(&root, testSet, batchResult);
653
654 {
655 string err;
656
657 if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR)
658 throw xe::Error(err);
659 }
660 }
661
662 } // anonymous
663
main(int argc,const char * const * argv)664 int main (int argc, const char* const* argv)
665 {
666 CommandLine cmdLine;
667
668 if (!parseCommandLine(cmdLine, argc, argv))
669 return -1;
670
671 try
672 {
673 runExecutor(cmdLine);
674 }
675 catch (const std::exception& e)
676 {
677 printf("%s\n", e.what());
678 return -1;
679 }
680
681 return 0;
682 }
683