test_results_tracker.cc revision 0f1bc08d4cfcc34181b0b5cbf065c40f687bf740
14e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Copyright 2013 The Chromium Authors. All rights reserved. 24e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 34e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// found in the LICENSE file. 44e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 54e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/test_results_tracker.h" 64e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 74e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/command_line.h" 84e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/file_util.h" 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/files/file_path.h" 104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/format_macros.h" 114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/json/json_file_value_serializer.h" 124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/logging.h" 134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/strings/stringprintf.h" 148bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)#include "base/test/launcher/test_launcher.h" 154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/values.h" 164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)namespace base { 184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ . 204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)using ::operator<<; 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)namespace { 234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// The default output file for XML output. 254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL( 264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "test_detail.xml"); 274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Utility function to print a list of test names. Uses iterator to be 294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// compatible with different containers, like vector and set. 304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)template<typename InputIterator> 314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void PrintTests(InputIterator first, 324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) InputIterator last, 334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) const std::string& description) { 344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) size_t count = std::distance(first, last); 354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (count == 0) 364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return; 374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, 394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "%" PRIuS " test%s %s:\n", 404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) count, 414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) count != 1 ? "s" : "", 424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) description.c_str()); 434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (InputIterator i = first; i != last; ++i) 444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, " %s\n", (*i).c_str()); 454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} // namespace 494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) { 514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)TestResultsTracker::~TestResultsTracker() { 544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!out_) 574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return; 584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); 594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\"" 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " disabled=\"\" errors=\"\" time=\"\">\n"); 611e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 621e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Maps test case names to test results. 631e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) typedef std::map<std::string, std::vector<TestResult> > TestCaseMap; 641e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestCaseMap test_case_map; 651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (PerIterationData::ResultsMap::iterator i = 674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_[iteration_].results.begin(); 684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) i != per_iteration_data_[iteration_].results.end(); 694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ++i) { 701e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Use the last test result as the final one. 711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestResult result = i->second.test_results.back(); 721e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_case_map[result.GetTestCaseName()].push_back(result); 731e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 741e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (TestCaseMap::iterator i = test_case_map.begin(); 751e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) i != test_case_map.end(); 761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ++i) { 774e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\"" 784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " disabled=\"\" errors=\"\" time=\"\">\n", 794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) i->first.c_str(), i->second.size()); 804e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (size_t j = 0; j < i->second.size(); ++j) { 814e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) const TestResult& result = i->second[j]; 824e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <testcase name=\"%s\" status=\"run\" time=\"%.3f\"" 834e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " classname=\"%s\">\n", 841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.GetTestName().c_str(), 854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result.elapsed_time.InSecondsF(), 861e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.GetTestCaseName().c_str()); 874e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (result.status != TestResult::TEST_SUCCESS) 884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <failure message=\"\" type=\"\"></failure>\n"); 894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " </testcase>\n"); 904e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " </testsuite>\n"); 924e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 934e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "</testsuites>\n"); 944e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fclose(out_); 954e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 964e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 974e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)bool TestResultsTracker::Init(const CommandLine& command_line) { 984e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 994e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Prevent initializing twice. 1014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (out_) { 1024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) NOTREACHED(); 1034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!command_line.HasSwitch(kGTestOutputFlag)) 1074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return true; 1084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag); 1104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) size_t colon_pos = flag.find(':'); 1114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath path; 1124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (colon_pos != std::string::npos) { 1134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath flag_path = 1144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) command_line.GetSwitchValuePath(kGTestOutputFlag); 1154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath::StringType path_string = flag_path.value(); 1164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = FilePath(path_string.substr(colon_pos + 1)); 1174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // If the given path ends with '/', consider it is a directory. 1184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Note: This does NOT check that a directory (or file) actually exists 1194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // (the behavior is same as what gtest does). 1204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (path.EndsWithSeparator()) { 1214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath executable = command_line.GetProgram().BaseName(); 1224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = path.Append(executable.ReplaceExtension( 1234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath::StringType(FILE_PATH_LITERAL("xml")))); 1244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (path.value().empty()) 1274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = FilePath(kDefaultOutputFile); 1284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath dir_name = path.DirName(); 1294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!DirectoryExists(dir_name)) { 1304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(WARNING) << "The output directory does not exist. " 1314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) << "Creating the directory: " << dir_name.value(); 1324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Create the directory if necessary (because the gtest does the same). 1334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!file_util::CreateDirectory(dir_name)) { 1344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(ERROR) << "Failed to created directory " << dir_name.value(); 1354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) out_ = file_util::OpenFile(path, "w"); 1394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!out_) { 1404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(ERROR) << "Cannot open output file: " 1414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) << path.value() << "."; 1424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return true; 1464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1488bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)void TestResultsTracker::OnTestIterationStarting() { 1494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Start with a fresh state for new iteration. 1524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) iteration_++; 1534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_.push_back(PerIterationData()); 1544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void TestResultsTracker::AddTestResult(const TestResult& result) { 1574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1591e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) per_iteration_data_[iteration_].results[ 1601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.full_name].test_results.push_back(result); 1618bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)} 1624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1638bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)void TestResultsTracker::PrintSummaryOfCurrentIteration() const { 1641e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::map<TestResult::Status, std::set<std::string> > tests_by_status; 1651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1661e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (PerIterationData::ResultsMap::const_iterator j = 1671e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) per_iteration_data_[iteration_].results.begin(); 1681e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) j != per_iteration_data_[iteration_].results.end(); 1691e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ++j) { 1701e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Use the last test result as the final one. 1711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestResult result = j->second.test_results.back(); 1721e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[result.status].insert(result.full_name); 1731e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 1741e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1751e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(), 1761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE].end(), 1771e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "failed"); 1781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(), 1791e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_TIMEOUT].end(), 1801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "timed out"); 1811e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(), 1821e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_CRASH].end(), 1831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "crashed"); 1841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(), 1851e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_SKIPPED].end(), 1861e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "skipped"); 1871e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(), 1881e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_UNKNOWN].end(), 1891e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "had unknown result"); 1904e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1924e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void TestResultsTracker::PrintSummaryOfAllIterations() const { 1934e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1944e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::map<TestResult::Status, std::set<std::string> > tests_by_status; 1964e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1974e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (int i = 0; i <= iteration_; i++) { 1981e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (PerIterationData::ResultsMap::const_iterator j = 1991e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) per_iteration_data_[i].results.begin(); 2001e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) j != per_iteration_data_[i].results.end(); 2014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ++j) { 2021e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Use the last test result as the final one. 2031e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestResult result = j->second.test_results.back(); 2041e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[result.status].insert(result.full_name); 2054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 2064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 2074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, "Summary of all itest iterations:\n"); 2094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 2104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2111e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(), 2121e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE].end(), 2134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "failed"); 2141e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(), 2151e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_TIMEOUT].end(), 2164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "timed out"); 2171e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(), 2181e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_CRASH].end(), 2194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "crashed"); 2201e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(), 2211e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_SKIPPED].end(), 2224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "skipped"); 2231e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(), 2241e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_UNKNOWN].end(), 2254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "had unknown result"); 2264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, "End of the summary.\n"); 2284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 2294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2310f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)void TestResultsTracker::AddGlobalTag(const std::string& tag) { 2320f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) global_tags_.insert(tag); 2330f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)} 2340f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const { 2364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) scoped_ptr<DictionaryValue> summary_root(new DictionaryValue); 2374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2380f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ListValue* global_tags = new ListValue; 2390f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) summary_root->Set("global_tags", global_tags); 2400f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2410f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) for (std::set<std::string>::const_iterator i = global_tags_.begin(); 2420f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) i != global_tags_.end(); 2430f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ++i) { 2440f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) global_tags->AppendString(*i); 2450f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) } 2460f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ListValue* per_iteration_data = new ListValue; 2484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) summary_root->Set("per_iteration_data", per_iteration_data); 2494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (int i = 0; i <= iteration_; i++) { 2511e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DictionaryValue* current_iteration_data = new DictionaryValue; 2524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data->Append(current_iteration_data); 2534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (PerIterationData::ResultsMap::const_iterator j = 2554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_[i].results.begin(); 2564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) j != per_iteration_data_[i].results.end(); 2574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ++j) { 2581e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ListValue* test_results = new ListValue; 2591e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) current_iteration_data->SetWithoutPathExpansion(j->first, test_results); 2601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2611e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (size_t k = 0; k < j->second.test_results.size(); k++) { 2621e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const TestResult& test_result = j->second.test_results[k]; 2634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DictionaryValue* test_result_value = new DictionaryValue; 2651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_results->Append(test_result_value); 2664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetString("status", test_result.StatusAsString()); 2684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetInteger( 2694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "elapsed_time_ms", test_result.elapsed_time.InMilliseconds()); 2704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetString("output_snippet", 2714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result.output_snippet); 2724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 2734e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 2744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 2754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) JSONFileValueSerializer serializer(path); 2774e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return serializer.Serialize(*summary_root); 2784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::AggregateTestResult::AggregateTestResult() { 2814e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2824e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::AggregateTestResult::~AggregateTestResult() { 2844e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2861e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::PerIterationData::PerIterationData() { 2871e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)} 2884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2891e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::PerIterationData::~PerIterationData() { 2904e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2924e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} // namespace base 293