executor/tools/xeCommandLineExecutor.cpp

/*-------------------------------------------------------------------------
 * drawElements Quality Program Test Executor
 * ------------------------------------------
 *
 * Copyright 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *//*!
 * \file
 * \brief Command line test executor.
 *//*--------------------------------------------------------------------*/

#include "xeBatchExecutor.hpp"
#include "xeLocalTcpIpLink.hpp"
#include "xeTcpIpLink.hpp"
#include "xeTestCaseListParser.hpp"
#include "xeTestLogWriter.hpp"
#include "xeTestResultParser.hpp"

#include "deCommandLine.hpp"
#include "deDirectoryIterator.hpp"
#include "deStringUtil.hpp"
#include "deUniquePtr.hpp"

#include "deString.h"

#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32)
#	include <signal.h>
#endif

using std::vector;
using std::string;

namespace
{

// Command line arguments.
namespace opt
{

DE_DECLARE_COMMAND_LINE_OPT(StartServer,	string);
DE_DECLARE_COMMAND_LINE_OPT(Host,			string);
DE_DECLARE_COMMAND_LINE_OPT(Port,			int);
DE_DECLARE_COMMAND_LINE_OPT(CaseListDir,	string);
DE_DECLARE_COMMAND_LINE_OPT(TestSet,		vector<string>);
DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet,		vector<string>);
DE_DECLARE_COMMAND_LINE_OPT(ContinueFile,	string);
DE_DECLARE_COMMAND_LINE_OPT(TestLogFile,	string);
DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile,	string);
DE_DECLARE_COMMAND_LINE_OPT(Summary,		bool);

// TargetConfiguration
DE_DECLARE_COMMAND_LINE_OPT(BinaryName,		string);
DE_DECLARE_COMMAND_LINE_OPT(WorkingDir,		string);
DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs,	string);

void parseCommaSeparatedList (const char* src, vector<string>* dst)
{
	std::istringstream	inStr	(src);
	string			comp;

	while (std::getline(inStr, comp, ','))
		dst->push_back(comp);
}

void registerOptions (de::cmdline::Parser& parser)
{
	using de::cmdline::Option;
	using de::cmdline::NamedValue;

	static const NamedValue<bool> s_yesNo[] =
	{
		{ "yes",	true	},
		{ "no",		false	}
	};

	parser << Option<StartServer>	("s",		"start-server",	"Start local execserver. Path to the execserver binary.")
		   << Option<Host>			("c",		"connect",		"Connect to host. Address of the execserver.")
		   << Option<Port>			("p",		"port",			"TCP port of the execserver.",											"50016")
		   << Option<CaseListDir>	("cd",		"caselistdir",	"Path to the directory containing test case XML files.",				".")
		   << Option<TestSet>		("t",		"testset",		"Comma-separated list of include filters.",								parseCommaSeparatedList)
		   << Option<ExcludeSet>	("e",		"exclude",		"Comma-separated list of exclude filters.",								parseCommaSeparatedList, "")
		   << Option<ContinueFile>	(DE_NULL,	"continue",		"Continue execution by initializing results from existing test log.")
		   << Option<TestLogFile>	("o",		"out",			"Output test log filename.",											"TestLog.qpa")
		   << Option<InfoLogFile>	("i",		"info",			"Output info log filename.",											"InfoLog.txt")
		   << Option<Summary>		(DE_NULL,	"summary",		"Print summary after running tests.",									s_yesNo, "yes")
		   << Option<BinaryName>	("b",		"binaryname",	"Test binary path. Relative to working directory.",						"<Unused>")
		   << Option<WorkingDir>	("wd",		"workdir",		"Working directory for the test execution.",							".")
		   << Option<CmdLineArgs>	(DE_NULL,	"cmdline",		"Additional command line arguments for the test binary.",				"");
}

} // opt

enum RunMode
{
	RUNMODE_CONNECT,
	RUNMODE_START_SERVER
};

struct CommandLine
{
	CommandLine (void)
		: port		(0)
		, summary	(false)
	{
	}

	xe::TargetConfiguration	targetCfg;
	RunMode					runMode;
	string					serverBinOrAddress;
	int						port;
	string					caseListDir;
	vector<string>			testset;
	vector<string>			exclude;
	string					inFile;
	string					outFile;
	string					infoFile;
	bool					summary;
};

bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
{
	de::cmdline::Parser			parser;
	de::cmdline::CommandLine	opts;

	XE_CHECK(argc >= 1);

	opt::registerOptions(parser);

	if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
	{
		std::cout << argv[0] << " [options]\n";
		parser.help(std::cout);
		return false;
	}

	if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>())
	{
		std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl;
		return false;
	}
	else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>())
	{
		std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl;
		return false;
	}

	if (!opts.hasOption<opt::TestSet>())
	{
		std::cout << "Invalid command line arguments. --testset not defined." << std::endl;
		return false;
	}

	if (opts.hasOption<opt::StartServer>())
	{
		cmdLine.runMode				= RUNMODE_START_SERVER;
		cmdLine.serverBinOrAddress	= opts.getOption<opt::StartServer>();
	}
	else
	{
		cmdLine.runMode				= RUNMODE_CONNECT;
		cmdLine.serverBinOrAddress	= opts.getOption<opt::Host>();
	}

	if (opts.hasOption<opt::ContinueFile>())
	{
		cmdLine.inFile = opts.getOption<opt::ContinueFile>();

		if (cmdLine.inFile.empty())
		{
			std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl;
			return false;
		}
	}

	cmdLine.port					= opts.getOption<opt::Port>();
	cmdLine.caseListDir				= opts.getOption<opt::CaseListDir>();
	cmdLine.testset					= opts.getOption<opt::TestSet>();
	cmdLine.exclude					= opts.getOption<opt::ExcludeSet>();
	cmdLine.outFile					= opts.getOption<opt::TestLogFile>();
	cmdLine.infoFile				= opts.getOption<opt::InfoLogFile>();
	cmdLine.summary					= opts.getOption<opt::Summary>();
	cmdLine.targetCfg.binaryName	= opts.getOption<opt::BinaryName>();
	cmdLine.targetCfg.workingDir	= opts.getOption<opt::WorkingDir>();
	cmdLine.targetCfg.cmdLineArgs	= opts.getOption<opt::CmdLineArgs>();

	return true;
}

bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
{
	int ptrnPos = 0;
	int casePos = 0;

	for (;;)
	{
		char c = casePath[casePos];
		char p = pattern[ptrnPos];

		if (p == '*')
		{
			/* Recurse to rest of positions. */
			int next = casePos;
			for (;;)
			{
				if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
					return DE_TRUE;

				if (casePath[next] == 0)
					return DE_FALSE; /* No match found. */
				else
					next += 1;
			}
			DE_ASSERT(DE_FALSE);
		}
		else if (c == 0 && p == 0)
			return true;
		else if (c == 0)
		{
			/* Incomplete match is ok for test groups. */
			return isTestGroup;
		}
		else if (c != p)
			return false;

		casePos += 1;
		ptrnPos += 1;
	}

	DE_ASSERT(false);
	return false;
}

void readCaseList (xe::TestGroup* root, const char* filename)
{
	xe::TestCaseListParser	caseListParser;
	std::ifstream			in				(filename, std::ios_base::binary);
	deUint8					buf[1024];

	XE_CHECK(in.good());

	caseListParser.init(root);

	for (;;)
	{
		in.read((char*)&buf[0], sizeof(buf));
		int numRead = (int)in.gcount();

		if (numRead > 0)
			caseListParser.parse(&buf[0], numRead);

		if (numRead < (int)sizeof(buf))
			break; // EOF
	}
}

void readCaseLists (xe::TestRoot& root, const char* caseListDir)
{
	int						testCaseListCount	= 0;
	de::DirectoryIterator	iter				(caseListDir);

	for (; iter.hasItem(); iter.next())
	{
		de::FilePath item = iter.getItem();

		if (item.getType() == de::FilePath::TYPE_FILE)
		{
			string baseName = item.getBaseName();
			if (baseName.find("-cases.xml") == baseName.length()-10)
			{
				string		packageName	= baseName.substr(0, baseName.length()-10);
				xe::TestGroup*	package		= root.createGroup(packageName.c_str(), "");

				readCaseList(package, item.getPath());
				testCaseListCount++;
			}
		}
	}

	if (testCaseListCount == 0)
		throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir)  + "'");
}

void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
{
	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
	{
		const xe::TestNode* child		= group.getChild(childNdx);
		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
		const string		fullPath	= child->getFullPath();

		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
		{
			if (isGroup)
			{
				// Recurse into group.
				addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
			}
			else
			{
				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
				testSet.add(child);
			}
		}
	}
}

void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
{
	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
	{
		const xe::TestNode* child		= group.getChild(childNdx);
		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
		const string		fullPath	= child->getFullPath();

		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
		{
			if (isGroup)
			{
				// Recurse into group.
				removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
			}
			else
			{
				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
				testSet.remove(child);
			}
		}
	}
}

class BatchResultHandler : public xe::TestLogHandler
{
public:
	BatchResultHandler (xe::BatchResult* batchResult)
		: m_batchResult(batchResult)
	{
	}

	void setSessionInfo (const xe::SessionInfo& sessionInfo)
	{
		m_batchResult->getSessionInfo() = sessionInfo;
	}

	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
	{
		// \todo [2012-11-01 pyry] What to do with duplicate results?
		if (m_batchResult->hasTestCaseResult(casePath))
			return m_batchResult->getTestCaseResult(casePath);
		else
			return m_batchResult->createTestCaseResult(casePath);
	}

	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
	{
	}

	void testCaseResultComplete (const xe::TestCaseResultPtr&)
	{
	}

private:
	xe::BatchResult* m_batchResult;
};

void readLogFile (xe::BatchResult* batchResult, const char* filename)
{
	std::ifstream		in		(filename, std::ifstream::binary|std::ifstream::in);
	BatchResultHandler	handler	(batchResult);
	xe::TestLogParser	parser	(&handler);
	deUint8				buf		[1024];
	int					numRead	= 0;

	for (;;)
	{
		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
		numRead = (int)in.gcount();

		if (numRead <= 0)
			break;

		parser.parse(&buf[0], numRead);
	}

	in.close();
}

void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
{
	int countByStatusCode[xe::TESTSTATUSCODE_LAST];
	std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);

	for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
	{
		const xe::TestNode* node = *iter;
		if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
		{
			const xe::TestCase*				testCase		= static_cast<const xe::TestCase*>(node);
			string							fullPath;
			xe::TestStatusCode				statusCode		= xe::TESTSTATUSCODE_PENDING;
			testCase->getFullPath(fullPath);

			// Parse result data if such exists.
			if (batchResult.hasTestCaseResult(fullPath.c_str()))
			{
				xe::ConstTestCaseResultPtr	resultData	= batchResult.getTestCaseResult(fullPath.c_str());
				xe::TestCaseResult			result;
				xe::TestResultParser		parser;

				xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
				statusCode = result.statusCode;
			}

			countByStatusCode[statusCode] += 1;
		}
	}

	printf("\nTest run summary:\n");
	int totalCases = 0;
	for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
	{
		if (countByStatusCode[code] > 0)
			printf("  %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);

		totalCases += countByStatusCode[code];
	}
	printf("  %20s: %5d\n", "Total", totalCases);
}

void writeInfoLog (const xe::InfoLog& log, const char* filename)
{
	std::ofstream out(filename, std::ios_base::binary);
	XE_CHECK(out.good());
	out.write((const char*)log.getBytes(), log.getSize());
	out.close();
}

xe::CommLink* createCommLink (const CommandLine& cmdLine)
{
	if (cmdLine.runMode == RUNMODE_START_SERVER)
	{
		xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
		try
		{
			link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port);
			return link;
		}
		catch (...)
		{
			delete link;
			throw;
		}
	}
	else if (cmdLine.runMode == RUNMODE_CONNECT)
	{
		de::SocketAddress address;

		address.setFamily(DE_SOCKETFAMILY_INET4);
		address.setProtocol(DE_SOCKETPROTOCOL_TCP);
		address.setHost(cmdLine.serverBinOrAddress.c_str());
		address.setPort(cmdLine.port);

		xe::TcpIpLink* link = new xe::TcpIpLink();
		try
		{
			std::string error;

			link->connect(address);
			return link;
		}
		catch (const std::exception& error)
		{
			delete link;
			throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what());
		}
		catch (...)
		{
			delete link;
			throw;
		}
	}
	else
	{
		DE_ASSERT(false);
		return DE_NULL;
	}
}

#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID)

static xe::BatchExecutor* s_executor = DE_NULL;

void signalHandler (int, siginfo_t*, void*)
{
	if (s_executor)
		s_executor->cancel();
}

void setupSignalHandler (xe::BatchExecutor* executor)
{
	s_executor = executor;
	struct sigaction sa;

	sa.sa_sigaction = signalHandler;
	sa.sa_flags = SA_SIGINFO | SA_RESTART;
	sigfillset(&sa.sa_mask);

	sigaction(SIGINT, &sa, DE_NULL);
}

void resetSignalHandler (void)
{
	struct sigaction sa;

	sa.sa_handler = SIG_DFL;
	sa.sa_flags = SA_RESTART;
	sigfillset(&sa.sa_mask);

	sigaction(SIGINT, &sa, DE_NULL);
	s_executor = DE_NULL;
}

#elif (DE_OS == DE_OS_WIN32)

static xe::BatchExecutor* s_executor = DE_NULL;

void signalHandler (int)
{
	if (s_executor)
		s_executor->cancel();
}

void setupSignalHandler (xe::BatchExecutor* executor)
{
	s_executor = executor;
	signal(SIGINT, signalHandler);
}

void resetSignalHandler (void)
{
	signal(SIGINT, SIG_DFL);
	s_executor = DE_NULL;
}

#else

void setupSignalHandler (xe::BatchExecutor*)
{
}

void resetSignalHandler (void)
{
}

#endif

void runExecutor (const CommandLine& cmdLine)
{
	xe::TestRoot root;

	// Read case list definitions.
	readCaseLists(root, cmdLine.caseListDir.c_str());

	// Build test set.
	xe::TestSet testSet;

	// Build test set.
	for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
		addMatchingCases(root, testSet, filterIter->c_str());

	if (testSet.empty())
		throw xe::Error("None of the test case lists contains tests matching any of the test sets.");

	// Remove excluded cases.
	for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
		removeMatchingCases(root, testSet, filterIter->c_str());

	// Initialize batch result.
	xe::BatchResult	batchResult;
	xe::InfoLog		infoLog;

	// Read existing results from input file (if supplied).
	if (!cmdLine.inFile.empty())
		readLogFile(&batchResult, cmdLine.inFile.c_str());

	// Initialize commLink.
	de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine));

	xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);

	try
	{
		setupSignalHandler(&executor);
		executor.run();
		resetSignalHandler();
	}
	catch (...)
	{
		resetSignalHandler();

		if (!cmdLine.outFile.empty())
		{
			xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
			printf("Test log written to %s\n", cmdLine.outFile.c_str());
		}

		if (!cmdLine.infoFile.empty())
		{
			writeInfoLog(infoLog, cmdLine.infoFile.c_str());
			printf("Info log written to %s\n", cmdLine.infoFile.c_str());
		}

		if (cmdLine.summary)
			printBatchResultSummary(&root, testSet, batchResult);

		throw;
	}

	if (!cmdLine.outFile.empty())
	{
		xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
		printf("Test log written to %s\n", cmdLine.outFile.c_str());
	}

	if (!cmdLine.infoFile.empty())
	{
		writeInfoLog(infoLog, cmdLine.infoFile.c_str());
		printf("Info log written to %s\n", cmdLine.infoFile.c_str());
	}

	if (cmdLine.summary)
		printBatchResultSummary(&root, testSet, batchResult);

	{
		string err;

		if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR)
			throw xe::Error(err);
	}
}

} // anonymous

int main (int argc, const char* const* argv)
{
	CommandLine cmdLine;

	if (!parseCommandLine(cmdLine, argc, argv))
		return -1;

	try
	{
		runExecutor(cmdLine);
	}
	catch (const std::exception& e)
	{
		printf("%s\n", e.what());
		return -1;
	}

	return 0;
}