xref: /external/deqp/executor/tools/xeTestLogCompare.cpp

/*-------------------------------------------------------------------------
 * drawElements Quality Program Test Executor
 * ------------------------------------------
 *
 * Copyright 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *//*!
 * \file
 * \brief Test log compare utility.
 *//*--------------------------------------------------------------------*/

#include "xeTestLogParser.hpp"
#include "xeTestResultParser.hpp"
#include "deFilePath.hpp"
#include "deString.h"
#include "deThread.hpp"
#include "deCommandLine.hpp"

#include <vector>
#include <string>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <set>
#include <map>

using std::vector;
using std::string;
using std::set;
using std::map;

enum OutputMode
{
	OUTPUTMODE_ALL = 0,
	OUTPUTMODE_DIFF,

	OUTPUTMODE_LAST
};

enum OutputFormat
{
	OUTPUTFORMAT_TEXT = 0,
	OUTPUTFORMAT_CSV,

	OUTPUTFORMAT_LAST
};

enum OutputValue
{
	OUTPUTVALUE_STATUS_CODE = 0,
	OUTPUTVALUE_STATUS_DETAILS,

	OUTPUTVALUE_LAST
};

namespace opt
{

DE_DECLARE_COMMAND_LINE_OPT(OutMode,	OutputMode);
DE_DECLARE_COMMAND_LINE_OPT(OutFormat,	OutputFormat);
DE_DECLARE_COMMAND_LINE_OPT(OutValue,	OutputValue);

static void registerOptions (de::cmdline::Parser& parser)
{
	using de::cmdline::Option;
	using de::cmdline::NamedValue;

	static const NamedValue<OutputMode> s_outputModes[] =
	{
		{ "all",	OUTPUTMODE_ALL	},
		{ "diff",	OUTPUTMODE_DIFF	}
	};
	static const NamedValue<OutputFormat> s_outputFormats[] =
	{
		{ "text",	OUTPUTFORMAT_TEXT	},
		{ "csv",	OUTPUTFORMAT_CSV	}
	};
	static const NamedValue<OutputValue> s_outputValues[] =
	{
		{ "code",		OUTPUTVALUE_STATUS_CODE		},
		{ "details",	OUTPUTVALUE_STATUS_DETAILS	}
	};

	parser << Option<OutFormat>		("f",	"format",		"Output format",	s_outputFormats,	"csv")
		   << Option<OutMode>		("m",	"mode",			"Output mode",		s_outputModes,		"all")
		   << Option<OutValue>		("v",	"value",		"Value to extract",	s_outputValues,		"code");
}

} // opt

struct CommandLine
{
	CommandLine (void)
		: outMode	(OUTPUTMODE_ALL)
		, outFormat	(OUTPUTFORMAT_CSV)
		, outValue	(OUTPUTVALUE_STATUS_CODE)
	{
	}

	OutputMode			outMode;
	OutputFormat		outFormat;
	OutputValue			outValue;
	vector<string>		filenames;
};

struct ShortBatchResult
{
	vector<xe::TestCaseResultHeader>	resultHeaders;
	map<string, int>					resultMap;
};

class ShortResultHandler : public xe::TestLogHandler
{
public:
	ShortResultHandler (ShortBatchResult& result)
		: m_result(result)
	{
	}

	void setSessionInfo (const xe::SessionInfo&)
	{
		// Ignored.
	}

	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
	{
		return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
	}

	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
	{
		// Ignored.
	}

	void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
	{
		xe::TestCaseResultHeader	header;
		int							caseNdx	= (int)m_result.resultHeaders.size();

		header.casePath			= caseData->getTestCasePath();
		header.caseType			= xe::TESTCASETYPE_SELF_VALIDATE;
		header.statusCode		= caseData->getStatusCode();
		header.statusDetails	= caseData->getStatusDetails();

		if (header.statusCode == xe::TESTSTATUSCODE_LAST)
		{
			xe::TestCaseResult fullResult;

			xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());

			header = xe::TestCaseResultHeader(fullResult);
		}

		// Insert into result list & map.
		m_result.resultHeaders.push_back(header);
		m_result.resultMap[header.casePath] = caseNdx;
	}

private:
	ShortBatchResult&		m_result;
	xe::TestResultParser	m_testResultParser;
};

static void readLogFile (ShortBatchResult& batchResult, const char* filename)
{
	std::ifstream		in				(filename, std::ifstream::binary|std::ifstream::in);
	ShortResultHandler	resultHandler	(batchResult);
	xe::TestLogParser	parser			(&resultHandler);
	deUint8				buf				[1024];
	int					numRead			= 0;

	for (;;)
	{
		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
		numRead = (int)in.gcount();

		if (numRead <= 0)
			break;

		parser.parse(&buf[0], numRead);
	}

	in.close();
}

class LogFileReader : public de::Thread
{
public:
	LogFileReader (ShortBatchResult& batchResult, const char* filename)
		: m_batchResult	(batchResult)
		, m_filename	(filename)
	{
	}

	void run (void)
	{
		readLogFile(m_batchResult, m_filename.c_str());
	}

private:
	ShortBatchResult&	m_batchResult;
	std::string			m_filename;
};

static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
{
	// \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
	set<string> addedCases;

	for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
	{
		for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
		{
			if (addedCases.find(caseIter->casePath) == addedCases.end())
			{
				cases.push_back(caseIter->casePath);
				addedCases.insert(caseIter->casePath);
			}
		}
	}
}

static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
{
	headers.resize(batchResults.size());

	for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
	{
		const ShortBatchResult&				batchResult	= batchResults[ndx];
		map<string, int>::const_iterator	resultPos	= batchResult.resultMap.find(casePath);

		if (resultPos != batchResult.resultMap.end())
			headers[ndx] = batchResult.resultHeaders[resultPos->second];
		else
		{
			headers[ndx].casePath	= casePath;
			headers[ndx].caseType	= xe::TESTCASETYPE_SELF_VALIDATE;
			headers[ndx].statusCode	= xe::TESTSTATUSCODE_LAST;
		}
	}
}

static const char* getStatusCodeName (xe::TestStatusCode code)
{
	if (code == xe::TESTSTATUSCODE_LAST)
		return "Missing";
	else
		return xe::getTestStatusCodeName(code);
}

static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
{
	vector<ShortBatchResult>	results;
	vector<string>				batchNames;
	bool						compareOk	= true;

	XE_CHECK(!cmdLine.filenames.empty());

	try
	{
		// Read in batch results
		results.resize(cmdLine.filenames.size());
		{
			std::vector<de::SharedPtr<LogFileReader> > readers;

			for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
			{
				readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
				readers.back()->start();
			}

			for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
			{
				readers[ndx]->join();

				// Use file name as batch name.
				batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
			}
		}

		// Compute unified case list.
		vector<string> caseList;
		computeCaseList(caseList, results);

		// Stats.
		int		numCases		= (int)caseList.size();
		int		numEqual		= 0;

		if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
		{
			dst << "TestCasePath";
			for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
				dst << "," << *nameIter;
			dst << "\n";
		}

		// Compare cases.
		for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
		{
			const string&						caseName	= *caseIter;
			vector<xe::TestCaseResultHeader>	headers;
			bool								allEqual	= true;

			getTestResultHeaders(headers, results, caseName.c_str());

			for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
			{
				if (iter->statusCode != headers[0].statusCode)
				{
					allEqual = false;
					break;
				}
			}

			if (allEqual)
				numEqual += 1;

			if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
			{
				if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
				{
					dst << caseName << "\n";
					for (int ndx = 0; ndx < (int)headers.size(); ndx++)
						dst << "  " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
					dst << "\n";
				}
				else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
				{
					dst << caseName;
					for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
						dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
					dst << "\n";
				}
			}
		}

		compareOk = numEqual == numCases;

		if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
		{
			dst << "  " << numEqual << " / " << numCases << " test case results match.\n";
			dst << "  Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
		}
	}
	catch (const std::exception& e)
	{
		printf("%s\n", e.what());
		compareOk = false;
	}

	return compareOk;
}

static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
{
	de::cmdline::Parser			parser;
	de::cmdline::CommandLine	opts;

	XE_CHECK(argc >= 1);

	opt::registerOptions(parser);

	if (!parser.parse(argc-1, &argv[1], &opts, std::cerr)	||
		opts.getArgs().empty())
	{
		std::cout << argv[0] << ": [options] [filenames]\n";
		parser.help(std::cout);
		return false;
	}

	cmdLine.outFormat	= opts.getOption<opt::OutFormat>();
	cmdLine.outMode		= opts.getOption<opt::OutMode>();
	cmdLine.outValue	= opts.getOption<opt::OutValue>();
	cmdLine.filenames	= opts.getArgs();

	return true;
}

int main (int argc, const char* const* argv)
{
	CommandLine cmdLine;

	if (!parseCommandLine(cmdLine, argc, argv))
		return -1;

	try
	{
		bool compareOk = runCompare(cmdLine, std::cout);
		return compareOk ? 0 : -1;
	}
	catch (const std::exception& e)
	{
		printf("FATAL ERROR: %s\n", e.what());
		return -1;
	}
}