unit_test_launcher.cc revision 1e9bf3e0803691d0a228da41fc608347b6db4340
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "base/test/launcher/unit_test_launcher.h" 6 7#include "base/bind.h" 8#include "base/callback_helpers.h" 9#include "base/command_line.h" 10#include "base/compiler_specific.h" 11#include "base/file_util.h" 12#include "base/files/scoped_temp_dir.h" 13#include "base/format_macros.h" 14#include "base/message_loop/message_loop.h" 15#include "base/stl_util.h" 16#include "base/strings/string_number_conversions.h" 17#include "base/strings/string_util.h" 18#include "base/sys_info.h" 19#include "base/test/gtest_xml_util.h" 20#include "base/test/launcher/parallel_test_launcher.h" 21#include "base/test/launcher/test_launcher.h" 22#include "base/test/test_switches.h" 23#include "base/test/test_timeouts.h" 24#include "base/threading/thread_checker.h" 25#include "testing/gtest/include/gtest/gtest.h" 26 27namespace base { 28 29namespace { 30 31// This constant controls how many tests are run in a single batch by default. 32const size_t kDefaultTestBatchLimit = 10; 33 34const char kHelpFlag[] = "help"; 35 36// Flag to enable the new launcher logic. 37// TODO(phajdan.jr): Remove it, http://crbug.com/236893 . 38const char kBraveNewTestLauncherFlag[] = "brave-new-test-launcher"; 39 40// Flag to run all tests in a single process. 41const char kSingleProcessTestsFlag[] = "single-process-tests"; 42 43void PrintUsage() { 44 fprintf(stdout, 45 "Runs tests using the gtest framework, each batch of tests being\n" 46 "run in their own process. Supported command-line flags:\n" 47 "\n" 48 " --single-process-tests\n" 49 " Runs the tests and the launcher in the same process. Useful\n" 50 " for debugging a specific test in a debugger.\n" 51 " --test-launcher-jobs=N\n" 52 " Sets the number of parallel test jobs to N.\n" 53 " --test-launcher-batch-limit=N\n" 54 " Sets the limit of test batch to run in a single process to N.\n" 55 " --gtest_filter=...\n" 56 " Runs a subset of tests (see --gtest_help for more info).\n" 57 " --help\n" 58 " Shows this message.\n" 59 " --gtest_help\n" 60 " Shows the gtest help message.\n"); 61 fflush(stdout); 62} 63 64// Returns command line for child GTest process based on the command line 65// of current process. |test_names| is a vector of test full names 66// (e.g. "A.B"), |output_file| is path to the GTest XML output file. 67CommandLine GetCommandLineForChildGTestProcess( 68 const std::vector<std::string>& test_names, 69 const base::FilePath& output_file) { 70 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 71 72 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 73 new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 74 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 75 new_cmd_line.AppendSwitch(kBraveNewTestLauncherFlag); 76 77 return new_cmd_line; 78} 79 80class UnitTestLauncherDelegate : public TestLauncherDelegate { 81 public: 82 UnitTestLauncherDelegate(size_t jobs, size_t batch_limit) 83 : parallel_launcher_(jobs), 84 batch_limit_(batch_limit) { 85 } 86 87 virtual ~UnitTestLauncherDelegate() { 88 DCHECK(thread_checker_.CalledOnValidThread()); 89 } 90 91 private: 92 struct GTestCallbackState { 93 TestLauncher* test_launcher; 94 std::vector<std::string> test_names; 95 FilePath output_file; 96 }; 97 98 virtual void OnTestIterationStarting() OVERRIDE { 99 // Nothing to do. 100 } 101 102 virtual std::string GetTestNameForFiltering( 103 const testing::TestCase* test_case, 104 const testing::TestInfo* test_info) OVERRIDE { 105 DCHECK(thread_checker_.CalledOnValidThread()); 106 107 return std::string(test_case->name()) + "." + test_info->name(); 108 } 109 110 virtual bool ShouldRunTest(const testing::TestCase* test_case, 111 const testing::TestInfo* test_info) OVERRIDE { 112 DCHECK(thread_checker_.CalledOnValidThread()); 113 114 // There is no additional logic to disable specific tests. 115 return true; 116 } 117 118 virtual size_t RunTests(TestLauncher* test_launcher, 119 const std::vector<std::string>& test_names) OVERRIDE { 120 DCHECK(thread_checker_.CalledOnValidThread()); 121 122 std::vector<std::string> batch; 123 for (size_t i = 0; i < test_names.size(); i++) { 124 batch.push_back(test_names[i]); 125 126 if (batch.size() >= batch_limit_) { 127 RunBatch(test_launcher, batch); 128 batch.clear(); 129 } 130 } 131 132 RunBatch(test_launcher, batch); 133 134 return test_names.size(); 135 } 136 137 virtual size_t RetryTests( 138 TestLauncher* test_launcher, 139 const std::vector<std::string>& test_names) OVERRIDE { 140 MessageLoop::current()->PostTask( 141 FROM_HERE, 142 Bind(&UnitTestLauncherDelegate::RunSerially, 143 Unretained(this), 144 test_launcher, 145 test_names)); 146 return test_names.size(); 147 } 148 149 void RunSerially(TestLauncher* test_launcher, 150 const std::vector<std::string>& test_names) { 151 if (test_names.empty()) 152 return; 153 154 std::vector<std::string> new_test_names(test_names); 155 std::string test_name(new_test_names.back()); 156 new_test_names.pop_back(); 157 158 // Create a dedicated temporary directory to store the xml result data 159 // per run to ensure clean state and make it possible to launch multiple 160 // processes in parallel. 161 base::FilePath output_file; 162 CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(), 163 &output_file)); 164 output_file = output_file.AppendASCII("test_results.xml"); 165 166 std::vector<std::string> current_test_names; 167 current_test_names.push_back(test_name); 168 CommandLine cmd_line( 169 GetCommandLineForChildGTestProcess(current_test_names, output_file)); 170 171 GTestCallbackState callback_state; 172 callback_state.test_launcher = test_launcher; 173 callback_state.test_names = current_test_names; 174 callback_state.output_file = output_file; 175 176 parallel_launcher_.LaunchChildGTestProcess( 177 cmd_line, 178 std::string(), 179 TestTimeouts::test_launcher_timeout(), 180 Bind(&UnitTestLauncherDelegate::SerialGTestCallback, 181 Unretained(this), 182 callback_state, 183 new_test_names)); 184 } 185 186 void RunBatch(TestLauncher* test_launcher, 187 const std::vector<std::string>& test_names) { 188 DCHECK(thread_checker_.CalledOnValidThread()); 189 190 if (test_names.empty()) 191 return; 192 193 // Create a dedicated temporary directory to store the xml result data 194 // per run to ensure clean state and make it possible to launch multiple 195 // processes in parallel. 196 base::FilePath output_file; 197 CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(), 198 &output_file)); 199 output_file = output_file.AppendASCII("test_results.xml"); 200 201 CommandLine cmd_line( 202 GetCommandLineForChildGTestProcess(test_names, output_file)); 203 204 // Adjust the timeout depending on how many tests we're running 205 // (note that e.g. the last batch of tests will be smaller). 206 // TODO(phajdan.jr): Consider an adaptive timeout, which can change 207 // depending on how many tests ran and how many remain. 208 // Note: do NOT parse child's stdout to do that, it's known to be 209 // unreliable (e.g. buffering issues can mix up the output). 210 base::TimeDelta timeout = 211 test_names.size() * TestTimeouts::test_launcher_timeout(); 212 213 GTestCallbackState callback_state; 214 callback_state.test_launcher = test_launcher; 215 callback_state.test_names = test_names; 216 callback_state.output_file = output_file; 217 218 parallel_launcher_.LaunchChildGTestProcess( 219 cmd_line, 220 std::string(), 221 timeout, 222 Bind(&UnitTestLauncherDelegate::GTestCallback, 223 Unretained(this), 224 callback_state)); 225 } 226 227 void GTestCallback(const GTestCallbackState& callback_state, 228 int exit_code, 229 const TimeDelta& elapsed_time, 230 bool was_timeout, 231 const std::string& output) { 232 DCHECK(thread_checker_.CalledOnValidThread()); 233 std::vector<std::string> tests_to_relaunch_after_interruption; 234 bool called_any_callbacks = 235 ProcessTestResults(callback_state.test_launcher, 236 callback_state.test_names, 237 callback_state.output_file, 238 output, 239 exit_code, 240 was_timeout, 241 &tests_to_relaunch_after_interruption); 242 243 RunBatch(callback_state.test_launcher, 244 tests_to_relaunch_after_interruption); 245 246 if (called_any_callbacks) 247 parallel_launcher_.ResetOutputWatchdog(); 248 249 // The temporary file's directory is also temporary. 250 DeleteFile(callback_state.output_file.DirName(), true); 251 } 252 253 void SerialGTestCallback(const GTestCallbackState& callback_state, 254 const std::vector<std::string>& test_names, 255 int exit_code, 256 const TimeDelta& elapsed_time, 257 bool was_timeout, 258 const std::string& output) { 259 DCHECK(thread_checker_.CalledOnValidThread()); 260 std::vector<std::string> tests_to_relaunch_after_interruption; 261 bool called_any_callbacks = 262 ProcessTestResults(callback_state.test_launcher, 263 callback_state.test_names, 264 callback_state.output_file, 265 output, 266 exit_code, 267 was_timeout, 268 &tests_to_relaunch_after_interruption); 269 270 // There is only one test, there cannot be other tests to relaunch 271 // due to a crash. 272 DCHECK(tests_to_relaunch_after_interruption.empty()); 273 274 if (called_any_callbacks) { 275 parallel_launcher_.ResetOutputWatchdog(); 276 } else { 277 // There is only one test, we should have called back with its result. 278 NOTREACHED(); 279 } 280 281 // The temporary file's directory is also temporary. 282 DeleteFile(callback_state.output_file.DirName(), true); 283 284 MessageLoop::current()->PostTask( 285 FROM_HERE, 286 Bind(&UnitTestLauncherDelegate::RunSerially, 287 Unretained(this), 288 callback_state.test_launcher, 289 test_names)); 290 } 291 292 static bool ProcessTestResults( 293 TestLauncher* test_launcher, 294 const std::vector<std::string>& test_names, 295 const base::FilePath& output_file, 296 const std::string& output, 297 int exit_code, 298 bool was_timeout, 299 std::vector<std::string>* tests_to_relaunch_after_interruption) { 300 std::vector<TestResult> test_results; 301 bool crashed = false; 302 bool have_test_results = 303 ProcessGTestOutput(output_file, &test_results, &crashed); 304 305 bool called_any_callback = false; 306 307 if (have_test_results) { 308 // TODO(phajdan.jr): Check for duplicates and mismatches between 309 // the results we got from XML file and tests we intended to run. 310 std::map<std::string, TestResult> results_map; 311 for (size_t i = 0; i < test_results.size(); i++) 312 results_map[test_results[i].full_name] = test_results[i]; 313 314 bool had_interrupted_test = false; 315 316 for (size_t i = 0; i < test_names.size(); i++) { 317 if (ContainsKey(results_map, test_names[i])) { 318 TestResult test_result = results_map[test_names[i]]; 319 if (test_result.status == TestResult::TEST_CRASH) { 320 had_interrupted_test = true; 321 322 if (was_timeout) { 323 // Fix up the test status: we forcibly kill the child process 324 // after the timeout, so from XML results it looks just like 325 // a crash. 326 test_result.status = TestResult::TEST_TIMEOUT; 327 } 328 } else if (test_result.status == TestResult::TEST_SUCCESS || 329 test_result.status == TestResult::TEST_FAILURE) { 330 // We run multiple tests in a batch with a timeout applied 331 // to the entire batch. It is possible that with other tests 332 // running quickly some tests take longer than the per-test timeout. 333 // For consistent handling of tests independent of order and other 334 // factors, mark them as timing out. 335 if (test_result.elapsed_time > 336 TestTimeouts::test_launcher_timeout()) { 337 test_result.status = TestResult::TEST_TIMEOUT; 338 } 339 } 340 test_result.output_snippet = 341 GetTestOutputSnippet(test_result, output); 342 test_launcher->OnTestFinished(test_result); 343 called_any_callback = true; 344 } else if (had_interrupted_test) { 345 tests_to_relaunch_after_interruption->push_back(test_names[i]); 346 } else { 347 // TODO(phajdan.jr): Explicitly pass the info that the test didn't 348 // run for a mysterious reason. 349 LOG(ERROR) << "no test result for " << test_names[i]; 350 TestResult test_result; 351 test_result.full_name = test_names[i]; 352 test_result.status = TestResult::TEST_UNKNOWN; 353 test_result.output_snippet = 354 GetTestOutputSnippet(test_result, output); 355 test_launcher->OnTestFinished(test_result); 356 called_any_callback = true; 357 } 358 } 359 360 // TODO(phajdan.jr): Handle the case where processing XML output 361 // indicates a crash but none of the test results is marked as crashing. 362 363 // TODO(phajdan.jr): Handle the case where the exit code is non-zero 364 // but results file indicates that all tests passed (e.g. crash during 365 // shutdown). 366 } else { 367 fprintf(stdout, 368 "Failed to get out-of-band test success data, " 369 "dumping full stdio below:\n%s\n", 370 output.c_str()); 371 fflush(stdout); 372 373 // We do not have reliable details about test results (parsing test 374 // stdout is known to be unreliable), apply the executable exit code 375 // to all tests. 376 // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 377 // individually. 378 for (size_t i = 0; i < test_names.size(); i++) { 379 TestResult test_result; 380 test_result.full_name = test_names[i]; 381 test_result.status = TestResult::TEST_UNKNOWN; 382 test_launcher->OnTestFinished(test_result); 383 called_any_callback = true; 384 } 385 } 386 387 return called_any_callback; 388 } 389 390 ThreadChecker thread_checker_; 391 392 ParallelTestLauncher parallel_launcher_; 393 394 // Maximum number of tests to run in a single batch. 395 size_t batch_limit_; 396}; 397 398bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 399 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 400 return true; 401 402 std::string switch_value = 403 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 404 if (!StringToInt(switch_value, result) || *result < 1) { 405 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 406 return false; 407 } 408 409 return true; 410} 411 412} // namespace 413 414int LaunchUnitTests(int argc, 415 char** argv, 416 const RunTestSuiteCallback& run_test_suite) { 417 CommandLine::Init(argc, argv); 418 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 419 CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 420 !CommandLine::ForCurrentProcess()->HasSwitch(kBraveNewTestLauncherFlag)) { 421 return run_test_suite.Run(); 422 } 423 424 if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 425 PrintUsage(); 426 return 0; 427 } 428 429 base::TimeTicks start_time(base::TimeTicks::Now()); 430 431 testing::InitGoogleTest(&argc, argv); 432 TestTimeouts::Initialize(); 433 434 int jobs = SysInfo::NumberOfProcessors(); 435 if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs)) 436 return 1; 437 438 int batch_limit = kDefaultTestBatchLimit; 439 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 440 return 1; 441 442 fprintf(stdout, 443 "Starting tests (using %d parallel jobs)...\n" 444 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 445 "own process. For debugging a test inside a debugger, use the\n" 446 "--gtest_filter=<your_test_name> flag along with\n" 447 "--single-process-tests.\n", jobs); 448 fflush(stdout); 449 450 MessageLoopForIO message_loop; 451 452 base::UnitTestLauncherDelegate delegate(jobs, batch_limit); 453 base::TestLauncher launcher(&delegate); 454 bool success = launcher.Run(argc, argv); 455 456 fprintf(stdout, 457 "Tests took %" PRId64 " seconds.\n", 458 (base::TimeTicks::Now() - start_time).InSeconds()); 459 fflush(stdout); 460 461 return (success ? 0 : 1); 462} 463 464} // namespace base 465