unit_test_launcher.cc revision 8bcbed890bc3ce4d7a057a8f32cab53fa534672e
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "base/test/launcher/unit_test_launcher.h" 6 7#include "base/bind.h" 8#include "base/callback_helpers.h" 9#include "base/command_line.h" 10#include "base/compiler_specific.h" 11#include "base/file_util.h" 12#include "base/files/scoped_temp_dir.h" 13#include "base/format_macros.h" 14#include "base/message_loop/message_loop.h" 15#include "base/stl_util.h" 16#include "base/strings/string_number_conversions.h" 17#include "base/strings/string_util.h" 18#include "base/sys_info.h" 19#include "base/test/gtest_xml_util.h" 20#include "base/test/launcher/parallel_test_launcher.h" 21#include "base/test/launcher/test_launcher.h" 22#include "base/test/test_switches.h" 23#include "base/test/test_timeouts.h" 24#include "base/threading/thread_checker.h" 25#include "testing/gtest/include/gtest/gtest.h" 26 27namespace base { 28 29namespace { 30 31// This constant controls how many tests are run in a single batch by default. 32const size_t kDefaultTestBatchLimit = 10; 33 34const char kHelpFlag[] = "help"; 35 36// Flag to enable the new launcher logic. 37// TODO(phajdan.jr): Remove it, http://crbug.com/236893 . 38const char kBraveNewTestLauncherFlag[] = "brave-new-test-launcher"; 39 40// Flag to run all tests in a single process. 41const char kSingleProcessTestsFlag[] = "single-process-tests"; 42 43void PrintUsage() { 44 fprintf(stdout, 45 "Runs tests using the gtest framework, each batch of tests being\n" 46 "run in their own process. Supported command-line flags:\n" 47 "\n" 48 " --single-process-tests\n" 49 " Runs the tests and the launcher in the same process. Useful\n" 50 " for debugging a specific test in a debugger.\n" 51 " --test-launcher-jobs=N\n" 52 " Sets the number of parallel test jobs to N.\n" 53 " --test-launcher-batch-limit=N\n" 54 " Sets the limit of test batch to run in a single process to N.\n" 55 " --gtest_filter=...\n" 56 " Runs a subset of tests (see --gtest_help for more info).\n" 57 " --help\n" 58 " Shows this message.\n" 59 " --gtest_help\n" 60 " Shows the gtest help message.\n"); 61 fflush(stdout); 62} 63 64// Returns command line for child GTest process based on the command line 65// of current process. |test_names| is a vector of test full names 66// (e.g. "A.B"), |output_file| is path to the GTest XML output file. 67CommandLine GetCommandLineForChildGTestProcess( 68 const std::vector<std::string>& test_names, 69 const base::FilePath& output_file) { 70 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 71 72 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 73 new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 74 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 75 new_cmd_line.AppendSwitch(kBraveNewTestLauncherFlag); 76 77 return new_cmd_line; 78} 79 80class UnitTestLauncherDelegate : public TestLauncherDelegate { 81 public: 82 UnitTestLauncherDelegate(size_t jobs, size_t batch_limit) 83 : parallel_launcher_(jobs), 84 batch_limit_(batch_limit) { 85 } 86 87 virtual ~UnitTestLauncherDelegate() { 88 DCHECK(thread_checker_.CalledOnValidThread()); 89 } 90 91 private: 92 struct TestLaunchInfo { 93 std::string GetFullName() const { 94 return test_case_name + "." + test_name; 95 } 96 97 std::string test_case_name; 98 std::string test_name; 99 TestResultCallback callback; 100 }; 101 102 virtual void OnTestIterationStarting() OVERRIDE { 103 // Nothing to do. 104 } 105 106 virtual std::string GetTestNameForFiltering( 107 const testing::TestCase* test_case, 108 const testing::TestInfo* test_info) OVERRIDE { 109 DCHECK(thread_checker_.CalledOnValidThread()); 110 111 return std::string(test_case->name()) + "." + test_info->name(); 112 } 113 114 virtual bool ShouldRunTest(const testing::TestCase* test_case, 115 const testing::TestInfo* test_info) OVERRIDE { 116 DCHECK(thread_checker_.CalledOnValidThread()); 117 118 // There is no additional logic to disable specific tests. 119 return true; 120 } 121 122 virtual void RunTest(const testing::TestCase* test_case, 123 const testing::TestInfo* test_info, 124 const TestResultCallback& callback) OVERRIDE { 125 DCHECK(thread_checker_.CalledOnValidThread()); 126 127 TestLaunchInfo launch_info; 128 launch_info.test_case_name = test_case->name(); 129 launch_info.test_name = test_info->name(); 130 launch_info.callback = callback; 131 tests_.push_back(launch_info); 132 133 // Run tests in batches no larger than the limit. 134 if (tests_.size() >= batch_limit_) 135 RunRemainingTests(); 136 } 137 138 virtual void RunRemainingTests() OVERRIDE { 139 DCHECK(thread_checker_.CalledOnValidThread()); 140 141 if (tests_.empty()) 142 return; 143 144 // Create a dedicated temporary directory to store the xml result data 145 // per run to ensure clean state and make it possible to launch multiple 146 // processes in parallel. 147 base::FilePath output_file; 148 CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(), 149 &output_file)); 150 output_file = output_file.AppendASCII("test_results.xml"); 151 152 std::vector<std::string> test_names; 153 for (size_t i = 0; i < tests_.size(); i++) 154 test_names.push_back(tests_[i].GetFullName()); 155 156 CommandLine cmd_line( 157 GetCommandLineForChildGTestProcess(test_names, output_file)); 158 159 // Adjust the timeout depending on how many tests we're running 160 // (note that e.g. the last batch of tests will be smaller). 161 // TODO(phajdan.jr): Consider an adaptive timeout, which can change 162 // depending on how many tests ran and how many remain. 163 // Note: do NOT parse child's stdout to do that, it's known to be 164 // unreliable (e.g. buffering issues can mix up the output). 165 base::TimeDelta timeout = 166 test_names.size() * TestTimeouts::test_launcher_timeout(); 167 168 parallel_launcher_.LaunchChildGTestProcess( 169 cmd_line, 170 std::string(), 171 timeout, 172 Bind(&UnitTestLauncherDelegate::GTestCallback, 173 base::Unretained(this), 174 tests_, 175 output_file)); 176 tests_.clear(); 177 } 178 179 void GTestCallback(const std::vector<TestLaunchInfo>& tests, 180 const FilePath& output_file, 181 int exit_code, 182 const TimeDelta& elapsed_time, 183 bool was_timeout, 184 const std::string& output) { 185 DCHECK(thread_checker_.CalledOnValidThread()); 186 std::vector<TestLaunchInfo> tests_to_relaunch_after_interruption; 187 bool called_any_callbacks = 188 ProcessTestResults(tests, 189 output_file, 190 output, 191 exit_code, 192 was_timeout, 193 &tests_to_relaunch_after_interruption); 194 195 for (size_t i = 0; i < tests_to_relaunch_after_interruption.size(); i++) 196 tests_.push_back(tests_to_relaunch_after_interruption[i]); 197 RunRemainingTests(); 198 199 if (called_any_callbacks) 200 parallel_launcher_.ResetOutputWatchdog(); 201 202 // The temporary file's directory is also temporary. 203 DeleteFile(output_file.DirName(), true); 204 } 205 206 static bool ProcessTestResults( 207 const std::vector<TestLaunchInfo>& tests, 208 const base::FilePath& output_file, 209 const std::string& output, 210 int exit_code, 211 bool was_timeout, 212 std::vector<TestLaunchInfo>* tests_to_relaunch_after_interruption) { 213 std::vector<TestResult> test_results; 214 bool crashed = false; 215 bool have_test_results = 216 ProcessGTestOutput(output_file, &test_results, &crashed); 217 218 bool called_any_callback = false; 219 220 if (have_test_results) { 221 // TODO(phajdan.jr): Check for duplicates and mismatches between 222 // the results we got from XML file and tests we intended to run. 223 std::map<std::string, TestResult> results_map; 224 for (size_t i = 0; i < test_results.size(); i++) 225 results_map[test_results[i].GetFullName()] = test_results[i]; 226 227 bool had_interrupted_test = false; 228 229 for (size_t i = 0; i < tests.size(); i++) { 230 if (ContainsKey(results_map, tests[i].GetFullName())) { 231 TestResult test_result = results_map[tests[i].GetFullName()]; 232 if (test_result.status == TestResult::TEST_CRASH) { 233 had_interrupted_test = true; 234 235 if (was_timeout) { 236 // Fix up the test status: we forcibly kill the child process 237 // after the timeout, so from XML results it looks just like 238 // a crash. 239 test_result.status = TestResult::TEST_TIMEOUT; 240 } 241 } else if (test_result.status == TestResult::TEST_SUCCESS || 242 test_result.status == TestResult::TEST_FAILURE) { 243 // We run multiple tests in a batch with a timeout applied 244 // to the entire batch. It is possible that with other tests 245 // running quickly some tests take longer than the per-test timeout. 246 // For consistent handling of tests independent of order and other 247 // factors, mark them as timing out. 248 if (test_result.elapsed_time > 249 TestTimeouts::test_launcher_timeout()) { 250 test_result.status = TestResult::TEST_TIMEOUT; 251 } 252 } 253 test_result.output_snippet = 254 GetTestOutputSnippet(test_result, output); 255 tests[i].callback.Run(test_result); 256 called_any_callback = true; 257 } else if (had_interrupted_test) { 258 tests_to_relaunch_after_interruption->push_back(tests[i]); 259 } else { 260 // TODO(phajdan.jr): Explicitly pass the info that the test didn't 261 // run for a mysterious reason. 262 LOG(ERROR) << "no test result for " << tests[i].GetFullName(); 263 TestResult test_result; 264 test_result.test_case_name = tests[i].test_case_name; 265 test_result.test_name = tests[i].test_name; 266 test_result.status = TestResult::TEST_UNKNOWN; 267 test_result.output_snippet = 268 GetTestOutputSnippet(test_result, output); 269 tests[i].callback.Run(test_result); 270 called_any_callback = true; 271 } 272 } 273 274 // TODO(phajdan.jr): Handle the case where processing XML output 275 // indicates a crash but none of the test results is marked as crashing. 276 277 // TODO(phajdan.jr): Handle the case where the exit code is non-zero 278 // but results file indicates that all tests passed (e.g. crash during 279 // shutdown). 280 } else { 281 fprintf(stdout, 282 "Failed to get out-of-band test success data, " 283 "dumping full stdio below:\n%s\n", 284 output.c_str()); 285 fflush(stdout); 286 287 // We do not have reliable details about test results (parsing test 288 // stdout is known to be unreliable), apply the executable exit code 289 // to all tests. 290 // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 291 // individually. 292 for (size_t i = 0; i < tests.size(); i++) { 293 TestResult test_result; 294 test_result.test_case_name = tests[i].test_case_name; 295 test_result.test_name = tests[i].test_name; 296 test_result.status = TestResult::TEST_UNKNOWN; 297 tests[i].callback.Run(test_result); 298 called_any_callback = true; 299 } 300 } 301 302 return called_any_callback; 303 } 304 305 ThreadChecker thread_checker_; 306 307 ParallelTestLauncher parallel_launcher_; 308 309 // Maximum number of tests to run in a single batch. 310 size_t batch_limit_; 311 312 std::vector<TestLaunchInfo> tests_; 313}; 314 315bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 316 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 317 return true; 318 319 std::string switch_value = 320 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 321 if (!StringToInt(switch_value, result) || *result < 1) { 322 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 323 return false; 324 } 325 326 return true; 327} 328 329} // namespace 330 331int LaunchUnitTests(int argc, 332 char** argv, 333 const RunTestSuiteCallback& run_test_suite) { 334 CommandLine::Init(argc, argv); 335 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 336 CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 337 !CommandLine::ForCurrentProcess()->HasSwitch(kBraveNewTestLauncherFlag)) { 338 return run_test_suite.Run(); 339 } 340 341 if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 342 PrintUsage(); 343 return 0; 344 } 345 346 base::TimeTicks start_time(base::TimeTicks::Now()); 347 348 testing::InitGoogleTest(&argc, argv); 349 TestTimeouts::Initialize(); 350 351 int jobs = SysInfo::NumberOfProcessors(); 352 if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs)) 353 return 1; 354 355 int batch_limit = kDefaultTestBatchLimit; 356 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 357 return 1; 358 359 fprintf(stdout, 360 "Starting tests (using %d parallel jobs)...\n" 361 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 362 "own process. For debugging a test inside a debugger, use the\n" 363 "--gtest_filter=<your_test_name> flag along with\n" 364 "--single-process-tests.\n", jobs); 365 fflush(stdout); 366 367 MessageLoopForIO message_loop; 368 369 base::UnitTestLauncherDelegate delegate(jobs, batch_limit); 370 base::TestLauncher launcher(&delegate); 371 bool success = launcher.Run(argc, argv); 372 373 fprintf(stdout, 374 "Tests took %" PRId64 " seconds.\n", 375 (base::TimeTicks::Now() - start_time).InSeconds()); 376 fflush(stdout); 377 378 return (success ? 0 : 1); 379} 380 381} // namespace base 382