1/*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Test log compare utility.
22 *//*--------------------------------------------------------------------*/
23
24#include "xeTestLogParser.hpp"
25#include "xeTestResultParser.hpp"
26#include "deFilePath.hpp"
27#include "deString.h"
28#include "deThread.hpp"
29#include "deCommandLine.hpp"
30
31#include <vector>
32#include <string>
33#include <cstdio>
34#include <cstdlib>
35#include <fstream>
36#include <iostream>
37#include <set>
38#include <map>
39
40using std::vector;
41using std::string;
42using std::set;
43using std::map;
44
45enum OutputMode
46{
47	OUTPUTMODE_ALL = 0,
48	OUTPUTMODE_DIFF,
49
50	OUTPUTMODE_LAST
51};
52
53enum OutputFormat
54{
55	OUTPUTFORMAT_TEXT = 0,
56	OUTPUTFORMAT_CSV,
57
58	OUTPUTFORMAT_LAST
59};
60
61enum OutputValue
62{
63	OUTPUTVALUE_STATUS_CODE = 0,
64	OUTPUTVALUE_STATUS_DETAILS,
65
66	OUTPUTVALUE_LAST
67};
68
69namespace opt
70{
71
72DE_DECLARE_COMMAND_LINE_OPT(OutMode,	OutputMode);
73DE_DECLARE_COMMAND_LINE_OPT(OutFormat,	OutputFormat);
74DE_DECLARE_COMMAND_LINE_OPT(OutValue,	OutputValue);
75
76static void registerOptions (de::cmdline::Parser& parser)
77{
78	using de::cmdline::Option;
79	using de::cmdline::NamedValue;
80
81	static const NamedValue<OutputMode> s_outputModes[] =
82	{
83		{ "all",	OUTPUTMODE_ALL	},
84		{ "diff",	OUTPUTMODE_DIFF	}
85	};
86	static const NamedValue<OutputFormat> s_outputFormats[] =
87	{
88		{ "text",	OUTPUTFORMAT_TEXT	},
89		{ "csv",	OUTPUTFORMAT_CSV	}
90	};
91	static const NamedValue<OutputValue> s_outputValues[] =
92	{
93		{ "code",		OUTPUTVALUE_STATUS_CODE		},
94		{ "details",	OUTPUTVALUE_STATUS_DETAILS	}
95	};
96
97	parser << Option<OutFormat>		("f",	"format",		"Output format",	s_outputFormats,	"csv")
98		   << Option<OutMode>		("m",	"mode",			"Output mode",		s_outputModes,		"all")
99		   << Option<OutValue>		("v",	"value",		"Value to extract",	s_outputValues,		"code");
100}
101
102} // opt
103
104struct CommandLine
105{
106	CommandLine (void)
107		: outMode	(OUTPUTMODE_ALL)
108		, outFormat	(OUTPUTFORMAT_CSV)
109		, outValue	(OUTPUTVALUE_STATUS_CODE)
110	{
111	}
112
113	OutputMode			outMode;
114	OutputFormat		outFormat;
115	OutputValue			outValue;
116	vector<string>		filenames;
117};
118
119struct ShortBatchResult
120{
121	vector<xe::TestCaseResultHeader>	resultHeaders;
122	map<string, int>					resultMap;
123};
124
125class ShortResultHandler : public xe::TestLogHandler
126{
127public:
128	ShortResultHandler (ShortBatchResult& result)
129		: m_result(result)
130	{
131	}
132
133	void setSessionInfo (const xe::SessionInfo&)
134	{
135		// Ignored.
136	}
137
138	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
139	{
140		return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
141	}
142
143	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
144	{
145		// Ignored.
146	}
147
148	void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
149	{
150		xe::TestCaseResultHeader	header;
151		int							caseNdx	= (int)m_result.resultHeaders.size();
152
153		header.casePath			= caseData->getTestCasePath();
154		header.caseType			= xe::TESTCASETYPE_SELF_VALIDATE;
155		header.statusCode		= caseData->getStatusCode();
156		header.statusDetails	= caseData->getStatusDetails();
157
158		if (header.statusCode == xe::TESTSTATUSCODE_LAST)
159		{
160			xe::TestCaseResult fullResult;
161
162			xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
163
164			header = xe::TestCaseResultHeader(fullResult);
165		}
166
167		// Insert into result list & map.
168		m_result.resultHeaders.push_back(header);
169		m_result.resultMap[header.casePath] = caseNdx;
170	}
171
172private:
173	ShortBatchResult&		m_result;
174	xe::TestResultParser	m_testResultParser;
175};
176
177static void readLogFile (ShortBatchResult& batchResult, const char* filename)
178{
179	std::ifstream		in				(filename, std::ifstream::binary|std::ifstream::in);
180	ShortResultHandler	resultHandler	(batchResult);
181	xe::TestLogParser	parser			(&resultHandler);
182	deUint8				buf				[1024];
183	int					numRead			= 0;
184
185	for (;;)
186	{
187		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
188		numRead = (int)in.gcount();
189
190		if (numRead <= 0)
191			break;
192
193		parser.parse(&buf[0], numRead);
194	}
195
196	in.close();
197}
198
199class LogFileReader : public de::Thread
200{
201public:
202	LogFileReader (ShortBatchResult& batchResult, const char* filename)
203		: m_batchResult	(batchResult)
204		, m_filename	(filename)
205	{
206	}
207
208	void run (void)
209	{
210		readLogFile(m_batchResult, m_filename.c_str());
211	}
212
213private:
214	ShortBatchResult&	m_batchResult;
215	std::string			m_filename;
216};
217
218static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
219{
220	// \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
221	set<string> addedCases;
222
223	for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
224	{
225		for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
226		{
227			if (addedCases.find(caseIter->casePath) == addedCases.end())
228			{
229				cases.push_back(caseIter->casePath);
230				addedCases.insert(caseIter->casePath);
231			}
232		}
233	}
234}
235
236static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
237{
238	headers.resize(batchResults.size());
239
240	for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
241	{
242		const ShortBatchResult&				batchResult	= batchResults[ndx];
243		map<string, int>::const_iterator	resultPos	= batchResult.resultMap.find(casePath);
244
245		if (resultPos != batchResult.resultMap.end())
246			headers[ndx] = batchResult.resultHeaders[resultPos->second];
247		else
248		{
249			headers[ndx].casePath	= casePath;
250			headers[ndx].caseType	= xe::TESTCASETYPE_SELF_VALIDATE;
251			headers[ndx].statusCode	= xe::TESTSTATUSCODE_LAST;
252		}
253	}
254}
255
256static const char* getStatusCodeName (xe::TestStatusCode code)
257{
258	if (code == xe::TESTSTATUSCODE_LAST)
259		return "Missing";
260	else
261		return xe::getTestStatusCodeName(code);
262}
263
264static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
265{
266	vector<ShortBatchResult>	results;
267	vector<string>				batchNames;
268	bool						compareOk	= true;
269
270	XE_CHECK(!cmdLine.filenames.empty());
271
272	try
273	{
274		// Read in batch results
275		results.resize(cmdLine.filenames.size());
276		{
277			std::vector<de::SharedPtr<LogFileReader> > readers;
278
279			for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
280			{
281				readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
282				readers.back()->start();
283			}
284
285			for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
286			{
287				readers[ndx]->join();
288
289				// Use file name as batch name.
290				batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
291			}
292		}
293
294		// Compute unified case list.
295		vector<string> caseList;
296		computeCaseList(caseList, results);
297
298		// Stats.
299		int		numCases		= (int)caseList.size();
300		int		numEqual		= 0;
301
302		if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
303		{
304			dst << "TestCasePath";
305			for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
306				dst << "," << *nameIter;
307			dst << "\n";
308		}
309
310		// Compare cases.
311		for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
312		{
313			const string&						caseName	= *caseIter;
314			vector<xe::TestCaseResultHeader>	headers;
315			bool								allEqual	= true;
316
317			getTestResultHeaders(headers, results, caseName.c_str());
318
319			for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
320			{
321				if (iter->statusCode != headers[0].statusCode)
322				{
323					allEqual = false;
324					break;
325				}
326			}
327
328			if (allEqual)
329				numEqual += 1;
330
331			if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
332			{
333				if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
334				{
335					dst << caseName << "\n";
336					for (int ndx = 0; ndx < (int)headers.size(); ndx++)
337						dst << "  " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
338					dst << "\n";
339				}
340				else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
341				{
342					dst << caseName;
343					for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
344						dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
345					dst << "\n";
346				}
347			}
348		}
349
350		compareOk = numEqual == numCases;
351
352		if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
353		{
354			dst << "  " << numEqual << " / " << numCases << " test case results match.\n";
355			dst << "  Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
356		}
357	}
358	catch (const std::exception& e)
359	{
360		printf("%s\n", e.what());
361		compareOk = false;
362	}
363
364	return compareOk;
365}
366
367static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
368{
369	de::cmdline::Parser			parser;
370	de::cmdline::CommandLine	opts;
371
372	XE_CHECK(argc >= 1);
373
374	opt::registerOptions(parser);
375
376	if (!parser.parse(argc-1, &argv[1], &opts, std::cerr)	||
377		opts.getArgs().empty())
378	{
379		std::cout << argv[0] << ": [options] [filenames]\n";
380		parser.help(std::cout);
381		return false;
382	}
383
384	cmdLine.outFormat	= opts.getOption<opt::OutFormat>();
385	cmdLine.outMode		= opts.getOption<opt::OutMode>();
386	cmdLine.outValue	= opts.getOption<opt::OutValue>();
387	cmdLine.filenames	= opts.getArgs();
388
389	return true;
390}
391
392int main (int argc, const char* const* argv)
393{
394	CommandLine cmdLine;
395
396	if (!parseCommandLine(cmdLine, argc, argv))
397		return -1;
398
399	try
400	{
401		bool compareOk = runCompare(cmdLine, std::cout);
402		return compareOk ? 0 : -1;
403	}
404	catch (const std::exception& e)
405	{
406		printf("FATAL ERROR: %s\n", e.what());
407		return -1;
408	}
409}
410