13c827367444ee418f129b2c238299f49d3264554Jarkko Poyry/*------------------------------------------------------------------------- 23c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * drawElements Quality Program Test Executor 33c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * ------------------------------------------ 43c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * 53c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * Copyright 2014 The Android Open Source Project 63c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * 73c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * Licensed under the Apache License, Version 2.0 (the "License"); 83c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * you may not use this file except in compliance with the License. 93c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * You may obtain a copy of the License at 103c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * 113c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * http://www.apache.org/licenses/LICENSE-2.0 123c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * 133c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * Unless required by applicable law or agreed to in writing, software 143c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * distributed under the License is distributed on an "AS IS" BASIS, 153c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 163c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * See the License for the specific language governing permissions and 173c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * limitations under the License. 183c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * 193c827367444ee418f129b2c238299f49d3264554Jarkko Poyry *//*! 203c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * \file 213c827367444ee418f129b2c238299f49d3264554Jarkko Poyry * \brief Test log compare utility. 223c827367444ee418f129b2c238299f49d3264554Jarkko Poyry *//*--------------------------------------------------------------------*/ 233c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 243c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "xeTestLogParser.hpp" 253c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "xeTestResultParser.hpp" 263c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "deFilePath.hpp" 273c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "deString.h" 283c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "deThread.hpp" 293c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include "deCommandLine.hpp" 303c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 313c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <vector> 323c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <string> 333c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <cstdio> 343c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <cstdlib> 353c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <fstream> 363c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <iostream> 373c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <set> 383c827367444ee418f129b2c238299f49d3264554Jarkko Poyry#include <map> 393c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 403c827367444ee418f129b2c238299f49d3264554Jarkko Poyryusing std::vector; 413c827367444ee418f129b2c238299f49d3264554Jarkko Poyryusing std::string; 423c827367444ee418f129b2c238299f49d3264554Jarkko Poyryusing std::set; 433c827367444ee418f129b2c238299f49d3264554Jarkko Poyryusing std::map; 443c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 453c827367444ee418f129b2c238299f49d3264554Jarkko Poyryenum OutputMode 463c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 473c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTMODE_ALL = 0, 483c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTMODE_DIFF, 493c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 503c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTMODE_LAST 513c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 523c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 533c827367444ee418f129b2c238299f49d3264554Jarkko Poyryenum OutputFormat 543c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 553c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTFORMAT_TEXT = 0, 563c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTFORMAT_CSV, 573c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 583c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTFORMAT_LAST 593c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 603c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 613c827367444ee418f129b2c238299f49d3264554Jarkko Poyryenum OutputValue 623c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 633c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTVALUE_STATUS_CODE = 0, 643c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTVALUE_STATUS_DETAILS, 653c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 663c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OUTPUTVALUE_LAST 673c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 683c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 693c827367444ee418f129b2c238299f49d3264554Jarkko Poyrynamespace opt 703c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 713c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 723c827367444ee418f129b2c238299f49d3264554Jarkko PoyryDE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode); 733c827367444ee418f129b2c238299f49d3264554Jarkko PoyryDE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat); 743c827367444ee418f129b2c238299f49d3264554Jarkko PoyryDE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue); 753c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 763c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic void registerOptions (de::cmdline::Parser& parser) 773c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 783c827367444ee418f129b2c238299f49d3264554Jarkko Poyry using de::cmdline::Option; 793c827367444ee418f129b2c238299f49d3264554Jarkko Poyry using de::cmdline::NamedValue; 803c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 813c827367444ee418f129b2c238299f49d3264554Jarkko Poyry static const NamedValue<OutputMode> s_outputModes[] = 823c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 833c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "all", OUTPUTMODE_ALL }, 843c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "diff", OUTPUTMODE_DIFF } 853c827367444ee418f129b2c238299f49d3264554Jarkko Poyry }; 863c827367444ee418f129b2c238299f49d3264554Jarkko Poyry static const NamedValue<OutputFormat> s_outputFormats[] = 873c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 883c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "text", OUTPUTFORMAT_TEXT }, 893c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "csv", OUTPUTFORMAT_CSV } 903c827367444ee418f129b2c238299f49d3264554Jarkko Poyry }; 913c827367444ee418f129b2c238299f49d3264554Jarkko Poyry static const NamedValue<OutputValue> s_outputValues[] = 923c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 933c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "code", OUTPUTVALUE_STATUS_CODE }, 943c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { "details", OUTPUTVALUE_STATUS_DETAILS } 953c827367444ee418f129b2c238299f49d3264554Jarkko Poyry }; 963c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 973c827367444ee418f129b2c238299f49d3264554Jarkko Poyry parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv") 983c827367444ee418f129b2c238299f49d3264554Jarkko Poyry << Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all") 993c827367444ee418f129b2c238299f49d3264554Jarkko Poyry << Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code"); 1003c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 1013c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1023c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} // opt 1033c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1043c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystruct CommandLine 1053c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 1063c827367444ee418f129b2c238299f49d3264554Jarkko Poyry CommandLine (void) 1073c827367444ee418f129b2c238299f49d3264554Jarkko Poyry : outMode (OUTPUTMODE_ALL) 1083c827367444ee418f129b2c238299f49d3264554Jarkko Poyry , outFormat (OUTPUTFORMAT_CSV) 1093c827367444ee418f129b2c238299f49d3264554Jarkko Poyry , outValue (OUTPUTVALUE_STATUS_CODE) 1103c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1113c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1123c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1133c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OutputMode outMode; 1143c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OutputFormat outFormat; 1153c827367444ee418f129b2c238299f49d3264554Jarkko Poyry OutputValue outValue; 1163c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<string> filenames; 1173c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 1183c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1193c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystruct ShortBatchResult 1203c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 1213c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<xe::TestCaseResultHeader> resultHeaders; 1223c827367444ee418f129b2c238299f49d3264554Jarkko Poyry map<string, int> resultMap; 1233c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 1243c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1253c827367444ee418f129b2c238299f49d3264554Jarkko Poyryclass ShortResultHandler : public xe::TestLogHandler 1263c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 1273c827367444ee418f129b2c238299f49d3264554Jarkko Poyrypublic: 1283c827367444ee418f129b2c238299f49d3264554Jarkko Poyry ShortResultHandler (ShortBatchResult& result) 1293c827367444ee418f129b2c238299f49d3264554Jarkko Poyry : m_result(result) 1303c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1313c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1323c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1333c827367444ee418f129b2c238299f49d3264554Jarkko Poyry void setSessionInfo (const xe::SessionInfo&) 1343c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1353c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Ignored. 1363c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1373c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1383c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::TestCaseResultPtr startTestCaseResult (const char* casePath) 1393c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1403c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath)); 1413c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1423c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1433c827367444ee418f129b2c238299f49d3264554Jarkko Poyry void testCaseResultUpdated (const xe::TestCaseResultPtr&) 1443c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1453c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Ignored. 1463c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1473c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1483c827367444ee418f129b2c238299f49d3264554Jarkko Poyry void testCaseResultComplete (const xe::TestCaseResultPtr& caseData) 1493c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1503c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::TestCaseResultHeader header; 1513c827367444ee418f129b2c238299f49d3264554Jarkko Poyry int caseNdx = (int)m_result.resultHeaders.size(); 1523c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1533c827367444ee418f129b2c238299f49d3264554Jarkko Poyry header.casePath = caseData->getTestCasePath(); 1543c827367444ee418f129b2c238299f49d3264554Jarkko Poyry header.caseType = xe::TESTCASETYPE_SELF_VALIDATE; 1553c827367444ee418f129b2c238299f49d3264554Jarkko Poyry header.statusCode = caseData->getStatusCode(); 1563c827367444ee418f129b2c238299f49d3264554Jarkko Poyry header.statusDetails = caseData->getStatusDetails(); 1573c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1583c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (header.statusCode == xe::TESTSTATUSCODE_LAST) 1593c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1603c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::TestCaseResult fullResult; 1613c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1623c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get()); 1633c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1643c827367444ee418f129b2c238299f49d3264554Jarkko Poyry header = xe::TestCaseResultHeader(fullResult); 1653c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1663c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1673c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Insert into result list & map. 1683c827367444ee418f129b2c238299f49d3264554Jarkko Poyry m_result.resultHeaders.push_back(header); 1693c827367444ee418f129b2c238299f49d3264554Jarkko Poyry m_result.resultMap[header.casePath] = caseNdx; 1703c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1713c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1723c827367444ee418f129b2c238299f49d3264554Jarkko Poyryprivate: 1733c827367444ee418f129b2c238299f49d3264554Jarkko Poyry ShortBatchResult& m_result; 1743c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::TestResultParser m_testResultParser; 1753c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 1763c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1773c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic void readLogFile (ShortBatchResult& batchResult, const char* filename) 1783c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 1793c827367444ee418f129b2c238299f49d3264554Jarkko Poyry std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); 1803c827367444ee418f129b2c238299f49d3264554Jarkko Poyry ShortResultHandler resultHandler (batchResult); 1813c827367444ee418f129b2c238299f49d3264554Jarkko Poyry xe::TestLogParser parser (&resultHandler); 1823c827367444ee418f129b2c238299f49d3264554Jarkko Poyry deUint8 buf [1024]; 1833c827367444ee418f129b2c238299f49d3264554Jarkko Poyry int numRead = 0; 1843c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1853c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (;;) 1863c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 1873c827367444ee418f129b2c238299f49d3264554Jarkko Poyry in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); 1883c827367444ee418f129b2c238299f49d3264554Jarkko Poyry numRead = (int)in.gcount(); 1893c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1903c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (numRead <= 0) 1913c827367444ee418f129b2c238299f49d3264554Jarkko Poyry break; 1923c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1933c827367444ee418f129b2c238299f49d3264554Jarkko Poyry parser.parse(&buf[0], numRead); 1943c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 1953c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1963c827367444ee418f129b2c238299f49d3264554Jarkko Poyry in.close(); 1973c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 1983c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 1993c827367444ee418f129b2c238299f49d3264554Jarkko Poyryclass LogFileReader : public de::Thread 2003c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 2013c827367444ee418f129b2c238299f49d3264554Jarkko Poyrypublic: 2023c827367444ee418f129b2c238299f49d3264554Jarkko Poyry LogFileReader (ShortBatchResult& batchResult, const char* filename) 2033c827367444ee418f129b2c238299f49d3264554Jarkko Poyry : m_batchResult (batchResult) 2043c827367444ee418f129b2c238299f49d3264554Jarkko Poyry , m_filename (filename) 2053c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2063c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2073c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2083c827367444ee418f129b2c238299f49d3264554Jarkko Poyry void run (void) 2093c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2103c827367444ee418f129b2c238299f49d3264554Jarkko Poyry readLogFile(m_batchResult, m_filename.c_str()); 2113c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2123c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2133c827367444ee418f129b2c238299f49d3264554Jarkko Poyryprivate: 2143c827367444ee418f129b2c238299f49d3264554Jarkko Poyry ShortBatchResult& m_batchResult; 2153c827367444ee418f129b2c238299f49d3264554Jarkko Poyry std::string m_filename; 2163c827367444ee418f129b2c238299f49d3264554Jarkko Poyry}; 2173c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2183c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults) 2193c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 2203c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely). 2213c827367444ee418f129b2c238299f49d3264554Jarkko Poyry set<string> addedCases; 2223c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2233c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++) 2243c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2253c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++) 2263c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2273c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (addedCases.find(caseIter->casePath) == addedCases.end()) 2283c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2293c827367444ee418f129b2c238299f49d3264554Jarkko Poyry cases.push_back(caseIter->casePath); 2303c827367444ee418f129b2c238299f49d3264554Jarkko Poyry addedCases.insert(caseIter->casePath); 2313c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2323c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2333c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2343c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 2353c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2363c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath) 2373c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 2383c827367444ee418f129b2c238299f49d3264554Jarkko Poyry headers.resize(batchResults.size()); 2393c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2403c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (int ndx = 0; ndx < (int)batchResults.size(); ndx++) 2413c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2423c827367444ee418f129b2c238299f49d3264554Jarkko Poyry const ShortBatchResult& batchResult = batchResults[ndx]; 2433c827367444ee418f129b2c238299f49d3264554Jarkko Poyry map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath); 2443c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2453c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (resultPos != batchResult.resultMap.end()) 2463c827367444ee418f129b2c238299f49d3264554Jarkko Poyry headers[ndx] = batchResult.resultHeaders[resultPos->second]; 2473c827367444ee418f129b2c238299f49d3264554Jarkko Poyry else 2483c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2493c827367444ee418f129b2c238299f49d3264554Jarkko Poyry headers[ndx].casePath = casePath; 2503c827367444ee418f129b2c238299f49d3264554Jarkko Poyry headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE; 2513c827367444ee418f129b2c238299f49d3264554Jarkko Poyry headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST; 2523c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2533c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2543c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 2553c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2563c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic const char* getStatusCodeName (xe::TestStatusCode code) 2573c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 2583c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (code == xe::TESTSTATUSCODE_LAST) 2593c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return "Missing"; 2603c827367444ee418f129b2c238299f49d3264554Jarkko Poyry else 2613c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return xe::getTestStatusCodeName(code); 2623c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 2633c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2643c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic bool runCompare (const CommandLine& cmdLine, std::ostream& dst) 2653c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 2663c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<ShortBatchResult> results; 2673c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<string> batchNames; 2683c827367444ee418f129b2c238299f49d3264554Jarkko Poyry bool compareOk = true; 2693c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2703c827367444ee418f129b2c238299f49d3264554Jarkko Poyry XE_CHECK(!cmdLine.filenames.empty()); 2713c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2723c827367444ee418f129b2c238299f49d3264554Jarkko Poyry try 2733c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2743c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Read in batch results 2753c827367444ee418f129b2c238299f49d3264554Jarkko Poyry results.resize(cmdLine.filenames.size()); 2763c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2773c827367444ee418f129b2c238299f49d3264554Jarkko Poyry std::vector<de::SharedPtr<LogFileReader> > readers; 2783c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2793c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) 2803c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2813c827367444ee418f129b2c238299f49d3264554Jarkko Poyry readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str()))); 2823c827367444ee418f129b2c238299f49d3264554Jarkko Poyry readers.back()->start(); 2833c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2843c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2853c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) 2863c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 2873c827367444ee418f129b2c238299f49d3264554Jarkko Poyry readers[ndx]->join(); 2883c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2893c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Use file name as batch name. 2903c827367444ee418f129b2c238299f49d3264554Jarkko Poyry batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName()); 2913c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2923c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 2933c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2943c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Compute unified case list. 2953c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<string> caseList; 2963c827367444ee418f129b2c238299f49d3264554Jarkko Poyry computeCaseList(caseList, results); 2973c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 2983c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Stats. 2993c827367444ee418f129b2c238299f49d3264554Jarkko Poyry int numCases = (int)caseList.size(); 3003c827367444ee418f129b2c238299f49d3264554Jarkko Poyry int numEqual = 0; 3013c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3023c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (cmdLine.outFormat == OUTPUTFORMAT_CSV) 3033c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3043c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "TestCasePath"; 3053c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++) 3063c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "," << *nameIter; 3073c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "\n"; 3083c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3093c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3103c827367444ee418f129b2c238299f49d3264554Jarkko Poyry // Compare cases. 3113c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++) 3123c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3133c827367444ee418f129b2c238299f49d3264554Jarkko Poyry const string& caseName = *caseIter; 3143c827367444ee418f129b2c238299f49d3264554Jarkko Poyry vector<xe::TestCaseResultHeader> headers; 3153c827367444ee418f129b2c238299f49d3264554Jarkko Poyry bool allEqual = true; 3163c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3173c827367444ee418f129b2c238299f49d3264554Jarkko Poyry getTestResultHeaders(headers, results, caseName.c_str()); 3183c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3193c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++) 3203c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3213c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (iter->statusCode != headers[0].statusCode) 3223c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3233c827367444ee418f129b2c238299f49d3264554Jarkko Poyry allEqual = false; 3243c827367444ee418f129b2c238299f49d3264554Jarkko Poyry break; 3253c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3263c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3273c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3283c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (allEqual) 3293c827367444ee418f129b2c238299f49d3264554Jarkko Poyry numEqual += 1; 3303c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3313c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual) 3323c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3333c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) 3343c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3353c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << caseName << "\n"; 3363c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (int ndx = 0; ndx < (int)headers.size(); ndx++) 3373c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n"; 3383c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "\n"; 3393c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3403c827367444ee418f129b2c238299f49d3264554Jarkko Poyry else if (cmdLine.outFormat == OUTPUTFORMAT_CSV) 3413c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3423c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << caseName; 3433c827367444ee418f129b2c238299f49d3264554Jarkko Poyry for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++) 3443c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str()); 3453c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << "\n"; 3463c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3473c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3483c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3493c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3503c827367444ee418f129b2c238299f49d3264554Jarkko Poyry compareOk = numEqual == numCases; 3513c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3523c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) 3533c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3543c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << " " << numEqual << " / " << numCases << " test case results match.\n"; 3553c827367444ee418f129b2c238299f49d3264554Jarkko Poyry dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n"; 3563c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3573c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3583c827367444ee418f129b2c238299f49d3264554Jarkko Poyry catch (const std::exception& e) 3593c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3603c827367444ee418f129b2c238299f49d3264554Jarkko Poyry printf("%s\n", e.what()); 3613c827367444ee418f129b2c238299f49d3264554Jarkko Poyry compareOk = false; 3623c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3633c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3643c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return compareOk; 3653c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 3663c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3673c827367444ee418f129b2c238299f49d3264554Jarkko Poyrystatic bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) 3683c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 3693c827367444ee418f129b2c238299f49d3264554Jarkko Poyry de::cmdline::Parser parser; 3703c827367444ee418f129b2c238299f49d3264554Jarkko Poyry de::cmdline::CommandLine opts; 3713c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3723c827367444ee418f129b2c238299f49d3264554Jarkko Poyry XE_CHECK(argc >= 1); 3733c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3743c827367444ee418f129b2c238299f49d3264554Jarkko Poyry opt::registerOptions(parser); 3753c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3763c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) || 3773c827367444ee418f129b2c238299f49d3264554Jarkko Poyry opts.getArgs().empty()) 3783c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 3793c827367444ee418f129b2c238299f49d3264554Jarkko Poyry std::cout << argv[0] << ": [options] [filenames]\n"; 3803c827367444ee418f129b2c238299f49d3264554Jarkko Poyry parser.help(std::cout); 3813c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return false; 3823c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 3833c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3843c827367444ee418f129b2c238299f49d3264554Jarkko Poyry cmdLine.outFormat = opts.getOption<opt::OutFormat>(); 3853c827367444ee418f129b2c238299f49d3264554Jarkko Poyry cmdLine.outMode = opts.getOption<opt::OutMode>(); 3863c827367444ee418f129b2c238299f49d3264554Jarkko Poyry cmdLine.outValue = opts.getOption<opt::OutValue>(); 3873c827367444ee418f129b2c238299f49d3264554Jarkko Poyry cmdLine.filenames = opts.getArgs(); 3883c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3893c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return true; 3903c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 3913c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3923c827367444ee418f129b2c238299f49d3264554Jarkko Poyryint main (int argc, const char* const* argv) 3933c827367444ee418f129b2c238299f49d3264554Jarkko Poyry{ 3943c827367444ee418f129b2c238299f49d3264554Jarkko Poyry CommandLine cmdLine; 3953c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3963c827367444ee418f129b2c238299f49d3264554Jarkko Poyry if (!parseCommandLine(cmdLine, argc, argv)) 3973c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return -1; 3983c827367444ee418f129b2c238299f49d3264554Jarkko Poyry 3993c827367444ee418f129b2c238299f49d3264554Jarkko Poyry try 4003c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 4013c827367444ee418f129b2c238299f49d3264554Jarkko Poyry bool compareOk = runCompare(cmdLine, std::cout); 4023c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return compareOk ? 0 : -1; 4033c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 4043c827367444ee418f129b2c238299f49d3264554Jarkko Poyry catch (const std::exception& e) 4053c827367444ee418f129b2c238299f49d3264554Jarkko Poyry { 4063c827367444ee418f129b2c238299f49d3264554Jarkko Poyry printf("FATAL ERROR: %s\n", e.what()); 4073c827367444ee418f129b2c238299f49d3264554Jarkko Poyry return -1; 4083c827367444ee418f129b2c238299f49d3264554Jarkko Poyry } 4093c827367444ee418f129b2c238299f49d3264554Jarkko Poyry} 410