14e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Copyright 2013 The Chromium Authors. All rights reserved. 24e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 34e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// found in the LICENSE file. 44e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 54e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/test_results_tracker.h" 64e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/base64.h" 84e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/command_line.h" 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/files/file_path.h" 106e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)#include "base/files/file_util.h" 114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/format_macros.h" 124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/json/json_file_value_serializer.h" 135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/json/string_escape.h" 144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/logging.h" 155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/strings/string_util.h" 164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/strings/stringprintf.h" 178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)#include "base/test/launcher/test_launcher.h" 184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/values.h" 194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)namespace base { 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)namespace { 234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// The default output file for XML output. 254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL( 264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "test_detail.xml"); 274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// Utility function to print a list of test names. Uses iterator to be 294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// compatible with different containers, like vector and set. 304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)template<typename InputIterator> 314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void PrintTests(InputIterator first, 324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) InputIterator last, 334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) const std::string& description) { 344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) size_t count = std::distance(first, last); 354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (count == 0) 364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return; 374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, 394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "%" PRIuS " test%s %s:\n", 404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) count, 414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) count != 1 ? "s" : "", 424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) description.c_str()); 434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (InputIterator i = first; i != last; ++i) 444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, " %s\n", (*i).c_str()); 454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)std::string TestNameWithoutDisabledPrefix(const std::string& test_name) { 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) std::string test_name_no_disabled(test_name); 505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ReplaceSubstringsAfterOffset(&test_name_no_disabled, 0, "DISABLED_", ""); 515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return test_name_no_disabled; 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} // namespace 554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) { 574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)TestResultsTracker::~TestResultsTracker() { 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!out_) 634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return; 644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); 654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\"" 664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " disabled=\"\" errors=\"\" time=\"\">\n"); 671e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 681e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Maps test case names to test results. 691e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) typedef std::map<std::string, std::vector<TestResult> > TestCaseMap; 701e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestCaseMap test_case_map; 711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (PerIterationData::ResultsMap::iterator i = 734e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_[iteration_].results.begin(); 744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) i != per_iteration_data_[iteration_].results.end(); 754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ++i) { 761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Use the last test result as the final one. 771e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestResult result = i->second.test_results.back(); 781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_case_map[result.GetTestCaseName()].push_back(result); 791e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (TestCaseMap::iterator i = test_case_map.begin(); 811e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) i != test_case_map.end(); 821e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ++i) { 834e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\"" 844e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " disabled=\"\" errors=\"\" time=\"\">\n", 854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) i->first.c_str(), i->second.size()); 864e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (size_t j = 0; j < i->second.size(); ++j) { 874e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) const TestResult& result = i->second[j]; 884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <testcase name=\"%s\" status=\"run\" time=\"%.3f\"" 894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) " classname=\"%s\">\n", 901e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.GetTestName().c_str(), 914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result.elapsed_time.InSecondsF(), 921e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.GetTestCaseName().c_str()); 934e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (result.status != TestResult::TEST_SUCCESS) 944e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " <failure message=\"\" type=\"\"></failure>\n"); 954e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " </testcase>\n"); 964e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 974e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, " </testsuite>\n"); 984e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 994e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(out_, "</testsuites>\n"); 1004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fclose(out_); 1014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)bool TestResultsTracker::Init(const CommandLine& command_line) { 1044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Prevent initializing twice. 1074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (out_) { 1084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) NOTREACHED(); 1094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!command_line.HasSwitch(kGTestOutputFlag)) 1134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return true; 1144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag); 1164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) size_t colon_pos = flag.find(':'); 1174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath path; 1184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (colon_pos != std::string::npos) { 1194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath flag_path = 1204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) command_line.GetSwitchValuePath(kGTestOutputFlag); 1214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath::StringType path_string = flag_path.value(); 1224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = FilePath(path_string.substr(colon_pos + 1)); 1234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // If the given path ends with '/', consider it is a directory. 1244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Note: This does NOT check that a directory (or file) actually exists 1254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // (the behavior is same as what gtest does). 1264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (path.EndsWithSeparator()) { 1274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath executable = command_line.GetProgram().BaseName(); 1284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = path.Append(executable.ReplaceExtension( 1294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath::StringType(FILE_PATH_LITERAL("xml")))); 1304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (path.value().empty()) 1334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) path = FilePath(kDefaultOutputFile); 1344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FilePath dir_name = path.DirName(); 1354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!DirectoryExists(dir_name)) { 1364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(WARNING) << "The output directory does not exist. " 1374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) << "Creating the directory: " << dir_name.value(); 1384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Create the directory if necessary (because the gtest does the same). 139a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if (!base::CreateDirectory(dir_name)) { 1404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(ERROR) << "Failed to created directory " << dir_name.value(); 1414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 144a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) out_ = OpenFile(path, "w"); 1454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if (!out_) { 1464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) LOG(ERROR) << "Cannot open output file: " 1474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) << path.value() << "."; 1484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return false; 1494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return true; 1524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1548bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)void TestResultsTracker::OnTestIterationStarting() { 1554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Start with a fresh state for new iteration. 1584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) iteration_++; 1594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_.push_back(PerIterationData()); 1604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)void TestResultsTracker::AddTest(const std::string& test_name) { 1635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Record disabled test names without DISABLED_ prefix so that they are easy 1645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // to compare with regular test names, e.g. before or after disabling. 1655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) all_tests_.insert(TestNameWithoutDisabledPrefix(test_name)); 1665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 1675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)void TestResultsTracker::AddDisabledTest(const std::string& test_name) { 1695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Record disabled test names without DISABLED_ prefix so that they are easy 1705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // to compare with regular test names, e.g. before or after disabling. 1715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name)); 1725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 1735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void TestResultsTracker::AddTestResult(const TestResult& result) { 1754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1771e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) per_iteration_data_[iteration_].results[ 1781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) result.full_name].test_results.push_back(result); 1798bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)} 1804e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1818bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)void TestResultsTracker::PrintSummaryOfCurrentIteration() const { 182f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestStatusMap tests_by_status(GetTestStatusMapForCurrentIteration()); 1831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(), 1851e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE].end(), 1861e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "failed"); 187f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(), 188f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(), 189f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) "failed on exit"); 1901e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(), 1911e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_TIMEOUT].end(), 1921e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "timed out"); 1931e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(), 1941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_CRASH].end(), 1951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "crashed"); 1961e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(), 1971e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_SKIPPED].end(), 1981e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "skipped"); 1991e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(), 2001e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_UNKNOWN].end(), 2011e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) "had unknown result"); 2024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void TestResultsTracker::PrintSummaryOfAllIterations() const { 2054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 2064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 207f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestStatusMap tests_by_status(GetTestStatusMapForAllIterations()); 2084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fprintf(stdout, "Summary of all test iterations:\n"); 2104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 2114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2121e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(), 2131e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE].end(), 2144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "failed"); 215f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(), 216f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(), 217f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) "failed on exit"); 2181e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(), 2191e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_TIMEOUT].end(), 2204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "timed out"); 2211e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(), 2221e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_CRASH].end(), 2234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "crashed"); 2241e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(), 2251e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_SKIPPED].end(), 2264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "skipped"); 2271e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(), 2281e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) tests_by_status[TestResult::TEST_UNKNOWN].end(), 2294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "had unknown result"); 2304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fprintf(stdout, "End of the summary.\n"); 2324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) fflush(stdout); 2334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 2344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2350f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)void TestResultsTracker::AddGlobalTag(const std::string& tag) { 2360f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) global_tags_.insert(tag); 2370f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles)} 2380f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const { 2404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) scoped_ptr<DictionaryValue> summary_root(new DictionaryValue); 2414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2420f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ListValue* global_tags = new ListValue; 2430f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) summary_root->Set("global_tags", global_tags); 2440f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2450f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) for (std::set<std::string>::const_iterator i = global_tags_.begin(); 2460f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) i != global_tags_.end(); 2470f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ++i) { 2480f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) global_tags->AppendString(*i); 2490f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) } 2500f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) 2515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ListValue* all_tests = new ListValue; 2525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) summary_root->Set("all_tests", all_tests); 2535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for (std::set<std::string>::const_iterator i = all_tests_.begin(); 2555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) i != all_tests_.end(); 2565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ++i) { 2575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) all_tests->AppendString(*i); 2585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 2595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ListValue* disabled_tests = new ListValue; 2615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) summary_root->Set("disabled_tests", disabled_tests); 2625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for (std::set<std::string>::const_iterator i = disabled_tests_.begin(); 2645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) i != disabled_tests_.end(); 2655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ++i) { 2665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) disabled_tests->AppendString(*i); 2675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 2685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ListValue* per_iteration_data = new ListValue; 2704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) summary_root->Set("per_iteration_data", per_iteration_data); 2714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (int i = 0; i <= iteration_; i++) { 2731e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DictionaryValue* current_iteration_data = new DictionaryValue; 2744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data->Append(current_iteration_data); 2754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (PerIterationData::ResultsMap::const_iterator j = 2774e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) per_iteration_data_[i].results.begin(); 2784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) j != per_iteration_data_[i].results.end(); 2794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ++j) { 2801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ListValue* test_results = new ListValue; 2811e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) current_iteration_data->SetWithoutPathExpansion(j->first, test_results); 2821e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (size_t k = 0; k < j->second.test_results.size(); k++) { 2841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const TestResult& test_result = j->second.test_results[k]; 2854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2864e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DictionaryValue* test_result_value = new DictionaryValue; 2871e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_results->Append(test_result_value); 2884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 2894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetString("status", test_result.StatusAsString()); 2904e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetInteger( 2914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) "elapsed_time_ms", test_result.elapsed_time.InMilliseconds()); 2925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // There are no guarantees about character encoding of the output 2945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // snippet. Escape it and record whether it was losless. 2955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // It's useful to have the output snippet as string in the summary 2965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // for easy viewing. 2975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) std::string escaped_output_snippet; 2985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bool losless_snippet = EscapeJSONString( 2995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_result.output_snippet, false, &escaped_output_snippet); 3004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result_value->SetString("output_snippet", 3015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) escaped_output_snippet); 3025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_result_value->SetBoolean("losless_snippet", losless_snippet); 3035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Also include the raw version (base64-encoded so that it can be safely 3055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // JSON-serialized - there are no guarantees about character encoding 3065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // of the snippet). This can be very useful piece of information when 3075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // debugging a test failure related to character encoding. 3085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) std::string base64_output_snippet; 3095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Base64Encode(test_result.output_snippet, &base64_output_snippet); 3105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_result_value->SetString("output_snippet_base64", 3115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) base64_output_snippet); 3124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 3134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 3144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 3154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) JSONFileValueSerializer serializer(path); 3174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return serializer.Serialize(*summary_root); 3184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 3194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 320f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)TestResultsTracker::TestStatusMap 321f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestResultsTracker::GetTestStatusMapForCurrentIteration() const { 322f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestStatusMap tests_by_status; 323f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) GetTestStatusForIteration(iteration_, &tests_by_status); 324f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return tests_by_status; 325f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 326f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 327f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)TestResultsTracker::TestStatusMap 328f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestResultsTracker::GetTestStatusMapForAllIterations() const { 329f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) TestStatusMap tests_by_status; 330f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for (int i = 0; i <= iteration_; i++) 331f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) GetTestStatusForIteration(i, &tests_by_status); 332f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return tests_by_status; 333f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 334f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 335f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void TestResultsTracker::GetTestStatusForIteration( 336f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) int iteration, TestStatusMap* map) const { 337f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for (PerIterationData::ResultsMap::const_iterator j = 338f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) per_iteration_data_[iteration].results.begin(); 339f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) j != per_iteration_data_[iteration].results.end(); 340f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) ++j) { 341f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // Use the last test result as the final one. 342f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) const TestResult& result = j->second.test_results.back(); 343f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) (*map)[result.status].insert(result.full_name); 344f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) } 345f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 346f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 3471e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::AggregateTestResult::AggregateTestResult() { 3484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 3494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3501e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::AggregateTestResult::~AggregateTestResult() { 3514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 3524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3531e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::PerIterationData::PerIterationData() { 3541e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)} 3554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3561e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)TestResultsTracker::PerIterationData::~PerIterationData() { 3574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 3584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} // namespace base 360