1/*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Command line test executor.
22 *//*--------------------------------------------------------------------*/
23
24#include "xeBatchExecutor.hpp"
25#include "xeTestCaseListParser.hpp"
26#include "xeTcpIpLink.hpp"
27#include "xeLocalTcpIpLink.hpp"
28#include "xeTestResultParser.hpp"
29#include "xeTestLogWriter.hpp"
30#include "deDirectoryIterator.hpp"
31#include "deCommandLine.hpp"
32#include "deString.h"
33
34#include <vector>
35#include <string>
36#include <cstdio>
37#include <cstdlib>
38#include <fstream>
39#include <memory>
40#include <algorithm>
41#include <iostream>
42#include <sstream>
43
44// Command line arguments.
45namespace opt
46{
47
48DE_DECLARE_COMMAND_LINE_OPT(StartServer,	std::string);
49DE_DECLARE_COMMAND_LINE_OPT(Host,			std::string);
50DE_DECLARE_COMMAND_LINE_OPT(Port,			int);
51DE_DECLARE_COMMAND_LINE_OPT(CaseListDir,	std::string);
52DE_DECLARE_COMMAND_LINE_OPT(TestSet,		std::vector<std::string>);
53DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet,		std::vector<std::string>);
54DE_DECLARE_COMMAND_LINE_OPT(ContinueFile,	std::string);
55DE_DECLARE_COMMAND_LINE_OPT(TestLogFile,	std::string);
56DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile,	std::string);
57DE_DECLARE_COMMAND_LINE_OPT(Summary,		bool);
58
59// TargetConfiguration
60DE_DECLARE_COMMAND_LINE_OPT(BinaryName,		std::string);
61DE_DECLARE_COMMAND_LINE_OPT(WorkingDir,		std::string);
62DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs,	std::string);
63
64static void parseCommaSeparatedList (const char* src, std::vector<std::string>* dst)
65{
66	std::istringstream	inStr	(src);
67	std::string			comp;
68
69	while (std::getline(inStr, comp, ','))
70		dst->push_back(comp);
71}
72
73void registerOptions (de::cmdline::Parser& parser)
74{
75	using de::cmdline::Option;
76	using de::cmdline::NamedValue;
77
78	static const NamedValue<bool> s_yesNo[] =
79	{
80		{ "yes",	true	},
81		{ "no",		false	}
82	};
83
84	parser << Option<StartServer>	("s",		"start-server",	"Start local execserver",								"")
85		   << Option<Host>			("c",		"connect",		"Connect to host",										"127.0.0.1")
86		   << Option<Port>			("p",		"port",			"Select TCP port to use",								"50016")
87		   << Option<CaseListDir>	("cd",		"caselistdir",	"Path to test case XML files",							".")
88		   << Option<TestSet>		("t",		"testset",		"Test set",												parseCommaSeparatedList,	"")
89		   << Option<ExcludeSet>	("e",		"exclude",		"Comma-separated list of exclude filters",				parseCommaSeparatedList,	"")
90		   << Option<ContinueFile>	(DE_NULL,	"continue",		"Continue execution by initializing results from existing test log")
91		   << Option<TestLogFile>	("o",		"out",			"Output test log filename",								"")
92		   << Option<InfoLogFile>	("i",		"info",			"Output info log filename",								"")
93		   << Option<Summary>		(DE_NULL,	"summary",		"Print summary at the end",								s_yesNo,	"yes")
94		   << Option<BinaryName>	("b",		"binaryname",	"Test binary path, relative to working directory",		"")
95		   << Option<WorkingDir>	("wd",		"workdir",		"Working directory for test execution",					"")
96		   << Option<CmdLineArgs>	(DE_NULL,	"cmdline",		"Additional command line arguments for test binary",	"");
97}
98
99} // opt
100
101using std::vector;
102using std::string;
103
104struct CommandLine
105{
106	CommandLine (void)
107		: port		(0)
108		, summary	(false)
109	{
110	}
111
112	xe::TargetConfiguration		targetCfg;
113	std::string					serverBin;
114	std::string					host;
115	int							port;
116	std::string					caseListDir;
117	std::vector<std::string>	testset;
118	std::vector<std::string>	exclude;
119	std::string					inFile;
120	std::string					outFile;
121	std::string					infoFile;
122	bool						summary;
123};
124
125static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
126{
127	de::cmdline::Parser			parser;
128	de::cmdline::CommandLine	opts;
129
130	XE_CHECK(argc >= 1);
131
132	opt::registerOptions(parser);
133
134	if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
135	{
136		std::cout << argv[0] << " [options]\n";
137		parser.help(std::cout);
138		return false;
139	}
140
141	cmdLine.serverBin				= opts.getOption<opt::StartServer>();
142	cmdLine.host					= opts.getOption<opt::Host>();
143	cmdLine.port					= opts.getOption<opt::Port>();
144	cmdLine.caseListDir				= opts.getOption<opt::CaseListDir>();
145	cmdLine.testset					= opts.getOption<opt::TestSet>();
146	cmdLine.exclude					= opts.getOption<opt::ExcludeSet>();
147	cmdLine.inFile					= opts.getOption<opt::ContinueFile>();
148	cmdLine.outFile					= opts.getOption<opt::TestLogFile>();
149	cmdLine.infoFile				= opts.getOption<opt::InfoLogFile>();
150	cmdLine.summary					= opts.getOption<opt::Summary>();
151	cmdLine.targetCfg.binaryName	= opts.getOption<opt::BinaryName>();
152	cmdLine.targetCfg.workingDir	= opts.getOption<opt::WorkingDir>();
153	cmdLine.targetCfg.cmdLineArgs	= opts.getOption<opt::CmdLineArgs>();
154
155	return true;
156}
157
158static bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
159{
160	int ptrnPos = 0;
161	int casePos = 0;
162
163	for (;;)
164	{
165		char c = casePath[casePos];
166		char p = pattern[ptrnPos];
167
168		if (p == '*')
169		{
170			/* Recurse to rest of positions. */
171			int next = casePos;
172			for (;;)
173			{
174				if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
175					return DE_TRUE;
176
177				if (casePath[next] == 0)
178					return DE_FALSE; /* No match found. */
179				else
180					next += 1;
181			}
182			DE_ASSERT(DE_FALSE);
183		}
184		else if (c == 0 && p == 0)
185			return true;
186		else if (c == 0)
187		{
188			/* Incomplete match is ok for test groups. */
189			return isTestGroup;
190		}
191		else if (c != p)
192			return false;
193
194		casePos += 1;
195		ptrnPos += 1;
196	}
197
198	DE_ASSERT(false);
199	return false;
200}
201
202static void readCaseList (xe::TestGroup* root, const char* filename)
203{
204	xe::TestCaseListParser	caseListParser;
205	std::ifstream			in				(filename, std::ios_base::binary);
206	deUint8					buf[1024];
207
208	XE_CHECK(in.good());
209
210	caseListParser.init(root);
211
212	for (;;)
213	{
214		in.read((char*)&buf[0], sizeof(buf));
215		int numRead = (int)in.gcount();
216
217		if (numRead > 0)
218			caseListParser.parse(&buf[0], numRead);
219
220		if (numRead < (int)sizeof(buf))
221			break; // EOF
222	}
223}
224
225static void readCaseLists (xe::TestRoot& root, const char* caseListDir)
226{
227	de::DirectoryIterator iter(caseListDir);
228
229	for (; iter.hasItem(); iter.next())
230	{
231		de::FilePath item = iter.getItem();
232
233		if (item.getType() == de::FilePath::TYPE_FILE)
234		{
235			std::string baseName = item.getBaseName();
236			if (baseName.find("-cases.xml") == baseName.length()-10)
237			{
238				std::string		packageName	= baseName.substr(0, baseName.length()-10);
239				xe::TestGroup*	package		= root.createGroup(packageName.c_str(), "");
240
241				readCaseList(package, item.getPath());
242			}
243		}
244	}
245}
246
247static void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
248{
249	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
250	{
251		const xe::TestNode* child		= group.getChild(childNdx);
252		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
253		const std::string	fullPath	= child->getFullPath();
254
255		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
256		{
257			if (isGroup)
258			{
259				// Recurse into group.
260				addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
261			}
262			else
263			{
264				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
265				testSet.add(child);
266			}
267		}
268	}
269}
270
271static void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
272{
273	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
274	{
275		const xe::TestNode* child		= group.getChild(childNdx);
276		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
277		const std::string	fullPath	= child->getFullPath();
278
279		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
280		{
281			if (isGroup)
282			{
283				// Recurse into group.
284				removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
285			}
286			else
287			{
288				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
289				testSet.remove(child);
290			}
291		}
292	}
293}
294
295class BatchResultHandler : public xe::TestLogHandler
296{
297public:
298	BatchResultHandler (xe::BatchResult* batchResult)
299		: m_batchResult(batchResult)
300	{
301	}
302
303	void setSessionInfo (const xe::SessionInfo& sessionInfo)
304	{
305		m_batchResult->getSessionInfo() = sessionInfo;
306	}
307
308	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
309	{
310		// \todo [2012-11-01 pyry] What to do with duplicate results?
311		if (m_batchResult->hasTestCaseResult(casePath))
312			return m_batchResult->getTestCaseResult(casePath);
313		else
314			return m_batchResult->createTestCaseResult(casePath);
315	}
316
317	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
318	{
319	}
320
321	void testCaseResultComplete (const xe::TestCaseResultPtr&)
322	{
323	}
324
325private:
326	xe::BatchResult* m_batchResult;
327};
328
329static void readLogFile (xe::BatchResult* batchResult, const char* filename)
330{
331	std::ifstream		in		(filename, std::ifstream::binary|std::ifstream::in);
332	BatchResultHandler	handler	(batchResult);
333	xe::TestLogParser	parser	(&handler);
334	deUint8				buf		[1024];
335	int					numRead	= 0;
336
337	for (;;)
338	{
339		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
340		numRead = (int)in.gcount();
341
342		if (numRead <= 0)
343			break;
344
345		parser.parse(&buf[0], numRead);
346	}
347
348	in.close();
349}
350
351static void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
352{
353	int countByStatusCode[xe::TESTSTATUSCODE_LAST];
354	std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
355
356	for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
357	{
358		const xe::TestNode* node = *iter;
359		if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
360		{
361			const xe::TestCase*				testCase		= static_cast<const xe::TestCase*>(node);
362			std::string						fullPath;
363			xe::TestStatusCode				statusCode		= xe::TESTSTATUSCODE_PENDING;
364			testCase->getFullPath(fullPath);
365
366			// Parse result data if such exists.
367			if (batchResult.hasTestCaseResult(fullPath.c_str()))
368			{
369				xe::ConstTestCaseResultPtr	resultData	= batchResult.getTestCaseResult(fullPath.c_str());
370				xe::TestCaseResult			result;
371				xe::TestResultParser		parser;
372
373				xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
374				statusCode = result.statusCode;
375			}
376
377			countByStatusCode[statusCode] += 1;
378		}
379	}
380
381	printf("\nTest run summary:\n");
382	int totalCases = 0;
383	for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
384	{
385		if (countByStatusCode[code] > 0)
386			printf("  %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
387
388		totalCases += countByStatusCode[code];
389	}
390	printf("  %20s: %5d\n", "Total", totalCases);
391}
392
393static void writeInfoLog (const xe::InfoLog& log, const char* filename)
394{
395	std::ofstream out(filename, std::ios_base::binary);
396	XE_CHECK(out.good());
397	out.write((const char*)log.getBytes(), log.getSize());
398	out.close();
399}
400
401static xe::CommLink* createCommLink (const CommandLine& cmdLine)
402{
403	if (!cmdLine.serverBin.empty())
404	{
405		xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
406		try
407		{
408			link->start(cmdLine.serverBin.c_str(), DE_NULL, cmdLine.port);
409			return link;
410		}
411		catch (...)
412		{
413			delete link;
414			throw;
415		}
416	}
417	else
418	{
419		de::SocketAddress address;
420		address.setFamily(DE_SOCKETFAMILY_INET4);
421		address.setProtocol(DE_SOCKETPROTOCOL_TCP);
422		address.setHost(cmdLine.host.c_str());
423		address.setPort(cmdLine.port);
424
425		xe::TcpIpLink* link = new xe::TcpIpLink();
426		try
427		{
428			link->connect(address);
429			return link;
430		}
431		catch (...)
432		{
433			delete link;
434			throw;
435		}
436	}
437}
438
439static void runExecutor (const CommandLine& cmdLine)
440{
441	xe::TestRoot root;
442
443	// Read case list definitions.
444	readCaseLists(root, cmdLine.caseListDir.c_str());
445
446	// Build test set.
447	xe::TestSet testSet;
448
449	// Build test set.
450	for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
451		addMatchingCases(root, testSet, filterIter->c_str());
452
453	// Remove excluded cases.
454	for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
455		removeMatchingCases(root, testSet, filterIter->c_str());
456
457	// Initialize batch result.
458	xe::BatchResult	batchResult;
459	xe::InfoLog		infoLog;
460
461	// Read existing results from input file (if supplied).
462	if (!cmdLine.inFile.empty())
463		readLogFile(&batchResult, cmdLine.inFile.c_str());
464
465	// Initialize commLink.
466	std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine));
467
468	xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
469	executor.run();
470
471	commLink.reset();
472
473	if (!cmdLine.outFile.empty())
474	{
475		xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
476		printf("Test log written to %s\n", cmdLine.outFile.c_str());
477	}
478
479	if (!cmdLine.infoFile.empty())
480	{
481		writeInfoLog(infoLog, cmdLine.infoFile.c_str());
482		printf("Info log written to %s\n", cmdLine.infoFile.c_str());
483	}
484
485	if (cmdLine.summary)
486		printBatchResultSummary(&root, testSet, batchResult);
487}
488
489int main (int argc, const char* const* argv)
490{
491	CommandLine cmdLine;
492
493	if (!parseCommandLine(cmdLine, argc, argv))
494		return -1;
495
496	try
497	{
498		runExecutor(cmdLine);
499	}
500	catch (const std::exception& e)
501	{
502		printf("%s\n", e.what());
503		return -1;
504	}
505
506	return 0;
507}
508