1/*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Class for executing tests.
22 *//*--------------------------------------------------------------------*/
23
24#include "tcuTestExecutor.hpp"
25#include "tcuCommandLine.hpp"
26#include "tcuPlatform.hpp"
27#include "tcuTestLog.hpp"
28
29#include "deInt32.h"
30
31#include <typeinfo>
32
33#include <stdio.h>
34#include <stdlib.h>
35#include <string.h>
36
37using std::string;
38using std::vector;
39
40namespace tcu
41{
42
43TestExecutor::TestExecutor (TestContext& testCtx, const CommandLine& cmdLine)
44	: m_testCtx				(testCtx)
45	, m_cmdLine				(cmdLine)
46	, m_rootNode			(DE_NULL)
47	, m_testCaseWrapper		(DE_NULL)
48	, m_testCaseListFile	(DE_NULL)
49	, m_testCaseListWriter	(DE_NULL)
50{
51	m_abortSession	= false;
52	m_isInTestCase	= false;
53
54	// Create the root node.
55	TestPackageRegistry*						packageRegistry	= TestPackageRegistry::getSingleton();
56	vector<TestPackageRegistry::PackageInfo*>	packageInfos	= packageRegistry->getPackageInfos();
57	vector<TestNode*>							testPackages;
58
59	for (int i = 0; i < (int)packageInfos.size(); i++)
60		testPackages.push_back(packageInfos[i]->createFunc(testCtx));
61
62	m_rootNode = new TestPackageRoot(testCtx, testPackages);
63
64	// Init traverse stack.
65	NodeIter iter(m_rootNode);
66	m_sessionStack.push_back(iter);
67}
68
69TestExecutor::~TestExecutor (void)
70{
71	if (m_testCaseListWriter)
72		qpXmlWriter_destroy(m_testCaseListWriter);
73
74	if (m_testCaseListFile)
75		fclose(m_testCaseListFile);
76
77	delete m_rootNode;
78}
79
80// Test sub-case iteration.
81void TestExecutor::enterTestPackage (TestPackage* testPackage, const char* packageName)
82{
83	DE_ASSERT(testPackage && packageName);
84
85	// Open file/writer for case dumping.
86	const RunMode runMode = m_cmdLine.getRunMode();
87	if (runMode == RUNMODE_DUMP_XML_CASELIST || runMode == RUNMODE_DUMP_TEXT_CASELIST)
88	{
89		const char* const	ext				= (runMode == RUNMODE_DUMP_XML_CASELIST) ? "xml" : "txt";
90		const string		fileName		= string(packageName) + "-cases." + ext;
91
92		print("Dumping all test case names in '%s' to file '%s'..\n", packageName, fileName.c_str());
93		TCU_CHECK(m_testCaseListFile = fopen(fileName.c_str(), "wb"));
94
95		if (runMode == RUNMODE_DUMP_XML_CASELIST)
96		{
97			TCU_CHECK(m_testCaseListWriter = qpXmlWriter_createFileWriter(m_testCaseListFile, DE_FALSE));
98
99			qpXmlWriter_startDocument(m_testCaseListWriter);
100			qpXmlWriter_startElement(m_testCaseListWriter, "TestCaseList", 0, DE_NULL);
101		}
102	}
103
104	// Initialize package.
105	testPackage->init();
106
107	// Store test case wrapper
108	m_testCaseWrapper = &testPackage->getTestCaseWrapper();
109	DE_ASSERT(m_testCaseWrapper);
110
111	// Set archive.
112	m_testCtx.setCurrentArchive(testPackage->getArchive());
113}
114
115void TestExecutor::leaveTestPackage (TestPackage* testPackage)
116{
117	DE_ASSERT(testPackage);
118
119	const RunMode runMode = m_cmdLine.getRunMode();
120	if (runMode == RUNMODE_DUMP_XML_CASELIST)
121	{
122		qpXmlWriter_endElement(m_testCaseListWriter, "TestCaseList");
123		qpXmlWriter_endDocument(m_testCaseListWriter);
124		qpXmlWriter_destroy(m_testCaseListWriter);
125		m_testCaseListWriter = DE_NULL;
126	}
127
128	if (runMode == RUNMODE_DUMP_TEXT_CASELIST || runMode == RUNMODE_DUMP_XML_CASELIST)
129	{
130		fclose(m_testCaseListFile);
131		m_testCaseListFile = DE_NULL;
132	}
133
134	DE_ASSERT(!m_testCaseListWriter && !m_testCaseListFile);
135
136	m_testCaseWrapper = DE_NULL;
137	m_testCtx.setCurrentArchive(m_testCtx.getRootArchive());
138
139	// Deinitialize package.
140	testPackage->deinit();
141}
142
143void TestExecutor::enterGroupNode (TestCaseGroup* testGroup, const char* casePath)
144{
145	DE_UNREF(casePath);
146	testGroup->init();
147}
148
149void TestExecutor::leaveGroupNode (TestCaseGroup* testGroup)
150{
151	testGroup->deinit();
152}
153
154static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
155{
156	switch (nodeType)
157	{
158		case NODETYPE_SELF_VALIDATE:	return QP_TEST_CASE_TYPE_SELF_VALIDATE;
159		case NODETYPE_PERFORMANCE:		return QP_TEST_CASE_TYPE_PERFORMANCE;
160		case NODETYPE_CAPABILITY:		return QP_TEST_CASE_TYPE_CAPABILITY;
161		case NODETYPE_ACCURACY:			return QP_TEST_CASE_TYPE_ACCURACY;
162		default:
163			DE_ASSERT(DE_FALSE);
164			return QP_TEST_CASE_TYPE_LAST;
165	}
166}
167
168bool TestExecutor::enterTestCase (TestCase* testCase, const char* casePath)
169{
170	const RunMode			runMode		= m_cmdLine.getRunMode();
171	const qpTestCaseType	caseType	= nodeTypeToTestCaseType(testCase->getNodeType());
172
173	if (runMode == RUNMODE_EXECUTE)
174	{
175		print("\nTest case '%s'..\n", casePath);
176
177		m_testCtx.getLog().startCase(casePath, caseType);
178		m_isInTestCase = true;
179		m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
180
181		if (!m_testCaseWrapper->initTestCase(testCase))
182		{
183			if (m_testCtx.getTestResult() == QP_TEST_RESULT_LAST)
184				m_testCtx.setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "Unexpected error in subcase init");
185			return false;
186		}
187	}
188
189	return true;
190}
191
192void TestExecutor::leaveTestCase (TestCase* testCase)
193{
194	const RunMode runMode = m_cmdLine.getRunMode();
195	if (runMode == RUNMODE_EXECUTE)
196	{
197		// De-init case.
198		const bool			deinitOk		= m_testCaseWrapper->deinitTestCase(testCase);
199		const qpTestResult	testResult		= m_testCtx.getTestResult();
200		const char* const	testResultDesc	= m_testCtx.getTestResultDesc();
201		const bool			terminateAfter	= m_testCtx.getTerminateAfter();
202		DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
203
204		m_isInTestCase = false;
205		m_testCtx.getLog().endCase(testResult, testResultDesc);
206
207		// Update statistics.
208		print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
209
210		m_result.numExecuted += 1;
211		switch (testResult)
212		{
213			case QP_TEST_RESULT_PASS:					m_result.numPassed			+= 1;	break;
214			case QP_TEST_RESULT_NOT_SUPPORTED:			m_result.numNotSupported	+= 1;	break;
215			case QP_TEST_RESULT_QUALITY_WARNING:		m_result.numWarnings		+= 1;	break;
216			case QP_TEST_RESULT_COMPATIBILITY_WARNING:	m_result.numWarnings		+= 1;	break;
217			default:									m_result.numFailed			+= 1;	break;
218		}
219
220		// terminateAfter, Resource error or any error in deinit means that execution should end
221		if (terminateAfter || !deinitOk || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
222			m_abortSession = true;
223
224		// \todo [2011-02-09 pyry] Disable watchdog temporarily?
225		if (m_testCtx.getWatchDog())
226			qpWatchDog_reset(m_testCtx.getWatchDog());
227	}
228}
229
230// Return true while session should still continue, false otherwise.
231bool TestExecutor::iterate (void)
232{
233	try
234	{
235		while (!m_sessionStack.empty())
236		{
237			// Get full path to node.
238			string nodePath = "";
239			for (int ndx = 0; ndx < (int)m_sessionStack.size(); ndx++)
240			{
241				NodeIter& iter = m_sessionStack[ndx];
242				if (ndx > 1) // ignore root package
243					nodePath += ".";
244				nodePath += iter.node->getName();
245			}
246
247			// Handle the node.
248			NodeIter& iter = m_sessionStack[m_sessionStack.size()-1];
249			DE_ASSERT(iter.node != DE_NULL);
250			TestNode*		node	= iter.node;
251			bool			isLeaf	= isTestNodeTypeExecutable(node->getNodeType());
252
253			switch (iter.getState())
254			{
255				case NodeIter::STATE_BEGIN:
256				{
257					// Return to parent if name doesn't match filter.
258					if (!(isLeaf ? m_cmdLine.checkTestCaseName(nodePath.c_str()) : m_cmdLine.checkTestGroupName(nodePath.c_str())))
259					{
260						m_sessionStack.pop_back();
261						break;
262					}
263
264					// Enter node.
265					bool enterOk = true;
266					switch (node->getNodeType())
267					{
268						case NODETYPE_ROOT:				/* nada */																	break;
269						case NODETYPE_PACKAGE:			enterTestPackage(static_cast<TestPackage*>(node), nodePath.c_str());		break;
270						case NODETYPE_GROUP:			enterGroupNode(static_cast<TestCaseGroup*>(node), nodePath.c_str());		break;
271						case NODETYPE_PERFORMANCE:
272						case NODETYPE_CAPABILITY:
273						case NODETYPE_ACCURACY:			/* fall-trough */
274						case NODETYPE_SELF_VALIDATE:	enterOk = enterTestCase(static_cast<TestCase*>(node), nodePath.c_str());	break;
275						default: DE_ASSERT(false);
276					}
277
278					if (m_cmdLine.getRunMode() == RUNMODE_EXECUTE)
279					{
280						if (isLeaf)
281						{
282							if (enterOk)
283								iter.setState(NodeIter::STATE_EXECUTE_TEST);
284							else
285								iter.setState(NodeIter::STATE_FINISH);
286						}
287						else
288						{
289							iter.setState(NodeIter::STATE_TRAVERSE_CHILDREN);
290						}
291					}
292					else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST)
293					{
294						if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE)
295						{
296							string			caseName	= iter.node->getName();
297							string			description	= iter.node->getDescription();
298							qpXmlAttribute	attribs[8];
299							int				numAttribs = 0;
300							const char*		caseType	= DE_NULL;
301
302							switch (node->getNodeType())
303							{
304								case NODETYPE_SELF_VALIDATE:	caseType = "SelfValidate";	break;
305								case NODETYPE_CAPABILITY:		caseType = "Capability";	break;
306								case NODETYPE_ACCURACY:			caseType = "Accuracy";		break;
307								case NODETYPE_PERFORMANCE:		caseType = "Performance";	break;
308								default:						caseType = "TestGroup";		break;
309							}
310
311							attribs[numAttribs++] = qpSetStringAttrib("Name", caseName.c_str());
312							attribs[numAttribs++] = qpSetStringAttrib("CaseType", caseType);
313							attribs[numAttribs++] = qpSetStringAttrib("Description", description.c_str());
314							qpXmlWriter_startElement(m_testCaseListWriter, "TestCase", numAttribs, attribs);
315						}
316
317						iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN);
318					}
319					else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_TEXT_CASELIST)
320					{
321						// \note Case list file is not open until we are in test package.
322						if (isLeaf)
323							fprintf(m_testCaseListFile, "TEST: %s\n", nodePath.c_str());
324						else if (node->getNodeType() != NODETYPE_ROOT)
325							fprintf(m_testCaseListFile, "GROUP: %s\n", nodePath.c_str());
326						iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN);
327					}
328
329					break;
330				}
331
332				case NodeIter::STATE_EXECUTE_TEST:
333				{
334					// Touch the watchdog.
335					m_testCtx.touchWatchdog();
336
337					// Iterate the sub-case.
338					TestCase::IterateResult iterateResult = m_testCaseWrapper->iterateTestCase(static_cast<TestCase*>(node));
339
340					if (iterateResult == TestCase::STOP)
341						iter.setState(NodeIter::STATE_FINISH);
342
343					return true; // return after each iteration (when another iteration follows).
344				}
345
346				case NodeIter::STATE_TRAVERSE_CHILDREN:
347				{
348					int numChildren = (int)iter.children.size();
349					if (++iter.curChildNdx < numChildren)
350					{
351						// Push child to stack.
352						TestNode* childNode = iter.children[iter.curChildNdx];
353						m_sessionStack.push_back(NodeIter(childNode));
354					}
355					else
356						iter.setState(NodeIter::STATE_FINISH);
357
358					break;
359				}
360
361				case NodeIter::STATE_FINISH:
362				{
363					if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST)
364					{
365						if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE)
366							qpXmlWriter_endElement(m_testCaseListWriter, "TestCase");
367					}
368
369					// Leave node.
370					switch (node->getNodeType())
371					{
372						case NODETYPE_ROOT:				/* nada */											break;
373						case NODETYPE_PACKAGE:			leaveTestPackage(static_cast<TestPackage*>(node));	break;
374						case NODETYPE_GROUP:			leaveGroupNode(static_cast<TestCaseGroup*>(node));	break;
375						case NODETYPE_ACCURACY:
376						case NODETYPE_CAPABILITY:
377						case NODETYPE_PERFORMANCE:		/* fall-thru */
378						case NODETYPE_SELF_VALIDATE:	leaveTestCase(static_cast<TestCase*>(node));		break;
379						default: DE_ASSERT(false);
380					}
381
382					m_sessionStack.pop_back();
383
384					// Return if execution should abort.
385					if (m_abortSession)
386						return false;
387
388					// Otherwise continue iterating.
389					break;
390				}
391
392				default:
393					DE_ASSERT(DE_FALSE);
394					break;
395			}
396		}
397	}
398	catch (const std::exception& e)
399	{
400		print("TestExecutor::iterateSession(): Caught unhandled %s: %s\n", typeid(e).name(), e.what());
401		throw;
402	}
403
404	m_result.isComplete = true;
405	return false;
406}
407
408} // tcu
409