test_launcher.cc revision 1e9bf3e0803691d0a228da41fc608347b6db4340
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/public/test/test_launcher.h" 6 7#include <map> 8#include <string> 9#include <vector> 10 11#include "base/command_line.h" 12#include "base/containers/hash_tables.h" 13#include "base/environment.h" 14#include "base/file_util.h" 15#include "base/files/scoped_temp_dir.h" 16#include "base/logging.h" 17#include "base/memory/linked_ptr.h" 18#include "base/memory/scoped_ptr.h" 19#include "base/message_loop/message_loop.h" 20#include "base/stl_util.h" 21#include "base/strings/string_number_conversions.h" 22#include "base/strings/string_util.h" 23#include "base/strings/utf_string_conversions.h" 24#include "base/test/launcher/parallel_test_launcher.h" 25#include "base/test/launcher/test_launcher.h" 26#include "base/test/test_suite.h" 27#include "base/test/test_switches.h" 28#include "base/test/test_timeouts.h" 29#include "base/time/time.h" 30#include "content/public/app/content_main.h" 31#include "content/public/app/content_main_delegate.h" 32#include "content/public/app/startup_helper_win.h" 33#include "content/public/common/content_switches.h" 34#include "content/public/common/sandbox_init.h" 35#include "content/public/test/browser_test.h" 36#include "net/base/escape.h" 37#include "testing/gtest/include/gtest/gtest.h" 38 39#if defined(OS_WIN) 40#include "base/base_switches.h" 41#include "content/common/sandbox_win.h" 42#include "sandbox/win/src/sandbox_factory.h" 43#include "sandbox/win/src/sandbox_types.h" 44#elif defined(OS_MACOSX) 45#include "base/mac/scoped_nsautorelease_pool.h" 46#endif 47 48namespace content { 49 50namespace { 51 52// Tests with this prefix run before the same test without it, and use the same 53// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests 54// that span browser restarts. 55const char kPreTestPrefix[] = "PRE_"; 56 57// Manual tests only run when --run-manual is specified. This allows writing 58// tests that don't run automatically but are still in the same test binary. 59// This is useful so that a team that wants to run a few tests doesn't have to 60// add a new binary that must be compiled on all builds. 61const char kManualTestPrefix[] = "MANUAL_"; 62 63TestLauncherDelegate* g_launcher_delegate; 64 65std::string RemoveAnyPrePrefixes(const std::string& test_name) { 66 std::string result(test_name); 67 ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string()); 68 return result; 69} 70 71void PrintUsage() { 72 fprintf(stdout, 73 "Runs tests using the gtest framework, each test being run in its own\n" 74 "process. Any gtest flags can be specified.\n" 75 " --single_process\n" 76 " Runs the tests and the launcher in the same process. Useful for \n" 77 " debugging a specific test in a debugger.\n" 78 " --single-process\n" 79 " Same as above, and also runs Chrome in single-process mode.\n" 80 " --help\n" 81 " Shows this message.\n" 82 " --gtest_help\n" 83 " Shows the gtest help message.\n"); 84} 85 86// Implementation of base::TestLauncherDelegate. This is also a test launcher, 87// wrapping a lower-level test launcher with content-specific code. 88class WrapperTestLauncherDelegate : public base::TestLauncherDelegate { 89 public: 90 WrapperTestLauncherDelegate(content::TestLauncherDelegate* launcher_delegate, 91 size_t jobs) 92 : launcher_delegate_(launcher_delegate), 93 timeout_count_(0), 94 printed_timeout_message_(false), 95 parallel_launcher_(jobs) { 96 CHECK(temp_dir_.CreateUniqueTempDir()); 97 } 98 99 // base::TestLauncherDelegate: 100 virtual void OnTestIterationStarting() OVERRIDE; 101 virtual std::string GetTestNameForFiltering( 102 const testing::TestCase* test_case, 103 const testing::TestInfo* test_info) OVERRIDE; 104 virtual bool ShouldRunTest(const testing::TestCase* test_case, 105 const testing::TestInfo* test_info) OVERRIDE; 106 virtual size_t RunTests(base::TestLauncher* test_launcher, 107 const std::vector<std::string>& test_names) OVERRIDE; 108 virtual size_t RetryTests( 109 base::TestLauncher* test_launcher, 110 const std::vector<std::string>& test_names) OVERRIDE; 111 112 private: 113 void DoRunTest(base::TestLauncher* test_launcher, 114 const std::string& test_name); 115 116 // Launches test named |test_name| using parallel launcher, 117 // given result of PRE_ test |pre_test_result|. 118 void RunDependentTest(base::TestLauncher* test_launcher, 119 const std::string test_name, 120 const base::TestResult& pre_test_result); 121 122 // Callback to receive result of a test. 123 void GTestCallback( 124 base::TestLauncher* test_launcher, 125 const std::string& test_name, 126 int exit_code, 127 const base::TimeDelta& elapsed_time, 128 bool was_timeout, 129 const std::string& output); 130 131 content::TestLauncherDelegate* launcher_delegate_; 132 133 // Number of times a test timeout occurred. 134 size_t timeout_count_; 135 136 // True after a message about too many timeouts has been printed, 137 // to avoid doing it more than once. 138 bool printed_timeout_message_; 139 140 base::ParallelTestLauncher parallel_launcher_; 141 142 // Store dependent test name (map is indexed by full test name). 143 typedef std::map<std::string, std::string> DependentTestMap; 144 DependentTestMap dependent_test_map_; 145 DependentTestMap reverse_dependent_test_map_; 146 147 // Store unique data directory prefix for test names (without PRE_ prefixes). 148 // PRE_ tests and tests that depend on them must share the same 149 // data directory. Using test name as directory name leads to too long 150 // names (exceeding UNIX_PATH_MAX, which creates a problem with 151 // process_singleton_linux). Create a randomly-named temporary directory 152 // and keep track of the names so that PRE_ tests can still re-use them. 153 typedef std::map<std::string, base::FilePath> UserDataDirMap; 154 UserDataDirMap user_data_dir_map_; 155 156 // Store names of all seen tests to properly handle PRE_ tests. 157 std::set<std::string> all_test_names_; 158 159 // Temporary directory for user data directories. 160 base::ScopedTempDir temp_dir_; 161 162 DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate); 163}; 164 165void WrapperTestLauncherDelegate::OnTestIterationStarting() { 166 dependent_test_map_.clear(); 167 user_data_dir_map_.clear(); 168} 169 170std::string WrapperTestLauncherDelegate::GetTestNameForFiltering( 171 const testing::TestCase* test_case, 172 const testing::TestInfo* test_info) { 173 return RemoveAnyPrePrefixes( 174 std::string(test_case->name()) + "." + test_info->name()); 175} 176 177bool WrapperTestLauncherDelegate::ShouldRunTest( 178 const testing::TestCase* test_case, 179 const testing::TestInfo* test_info) { 180 all_test_names_.insert( 181 std::string(test_case->name()) + "." + test_info->name()); 182 183 if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) && 184 !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) { 185 return false; 186 } 187 188 if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) { 189 // We will actually run PRE_ tests, but to ensure they run on the same shard 190 // as dependent tests, handle all these details internally. 191 return false; 192 } 193 194 // Stop test execution after too many timeouts. 195 if (timeout_count_ > 5) { 196 if (!printed_timeout_message_) { 197 printed_timeout_message_ = true; 198 printf("Too many timeouts, aborting test\n"); 199 } 200 return false; 201 } 202 203 return true; 204} 205 206std::string GetPreTestName(const std::string& full_name) { 207 size_t dot_pos = full_name.find('.'); 208 CHECK_NE(dot_pos, std::string::npos); 209 std::string test_case_name = full_name.substr(0, dot_pos); 210 std::string test_name = full_name.substr(dot_pos + 1); 211 return test_case_name + "." + kPreTestPrefix + test_name; 212} 213 214size_t WrapperTestLauncherDelegate::RunTests( 215 base::TestLauncher* test_launcher, 216 const std::vector<std::string>& test_names) { 217 // Number of additional tests to run because of dependencies. 218 size_t additional_tests_to_run_count = 0; 219 220 // Compute dependencies of tests to be run. 221 for (size_t i = 0; i < test_names.size(); i++) { 222 std::string full_name(test_names[i]); 223 std::string pre_test_name(GetPreTestName(full_name)); 224 225 while (ContainsKey(all_test_names_, pre_test_name)) { 226 additional_tests_to_run_count++; 227 228 DCHECK(!ContainsKey(dependent_test_map_, pre_test_name)); 229 dependent_test_map_[pre_test_name] = full_name; 230 231 DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name)); 232 reverse_dependent_test_map_[full_name] = pre_test_name; 233 234 full_name = pre_test_name; 235 pre_test_name = GetPreTestName(pre_test_name); 236 } 237 } 238 239 for (size_t i = 0; i < test_names.size(); i++) { 240 std::string full_name(test_names[i]); 241 242 // Make sure no PRE_ tests were requested explicitly. 243 DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name)); 244 245 if (!ContainsKey(user_data_dir_map_, full_name)) { 246 base::FilePath temp_dir; 247 CHECK(file_util::CreateTemporaryDirInDir( 248 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir)); 249 user_data_dir_map_[full_name] = temp_dir; 250 } 251 252 // If the test has any dependencies, get to the root and start with that. 253 while (ContainsKey(reverse_dependent_test_map_, full_name)) 254 full_name = GetPreTestName(full_name); 255 256 DoRunTest(test_launcher, full_name); 257 } 258 259 return test_names.size() + additional_tests_to_run_count; 260} 261 262size_t WrapperTestLauncherDelegate::RetryTests( 263 base::TestLauncher* test_launcher, 264 const std::vector<std::string>& test_names) { 265 // List of tests we can kick off right now, depending on no other tests. 266 std::vector<std::string> tests_to_run_now; 267 268 // We retry at least the tests requested to retry. 269 std::set<std::string> test_names_set(test_names.begin(), test_names.end()); 270 271 // In the face of PRE_ tests, we need to retry the entire chain of tests, 272 // from the very first one. 273 for (size_t i = 0; i < test_names.size(); i++) { 274 std::string test_name(test_names[i]); 275 while (ContainsKey(reverse_dependent_test_map_, test_name)) { 276 test_name = reverse_dependent_test_map_[test_name]; 277 test_names_set.insert(test_name); 278 } 279 } 280 281 // Discard user data directories from any previous runs. Start with 282 // fresh state. 283 for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin(); 284 i != user_data_dir_map_.end(); 285 ++i) { 286 // Delete temporary directories now to avoid using too much space in /tmp. 287 if (!base::DeleteFile(i->second, true)) { 288 LOG(WARNING) << "Failed to delete " << i->second.value(); 289 } 290 } 291 user_data_dir_map_.clear(); 292 293 for (std::set<std::string>::const_iterator i = test_names_set.begin(); 294 i != test_names_set.end(); 295 ++i) { 296 std::string full_name(*i); 297 298 // Make sure PRE_ tests and tests that depend on them share the same 299 // data directory - based it on the test name without prefixes. 300 std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name)); 301 if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) { 302 base::FilePath temp_dir; 303 CHECK(file_util::CreateTemporaryDirInDir( 304 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir)); 305 user_data_dir_map_[test_name_no_pre] = temp_dir; 306 } 307 308 size_t dot_pos = full_name.find('.'); 309 CHECK_NE(dot_pos, std::string::npos); 310 std::string test_case_name = full_name.substr(0, dot_pos); 311 std::string test_name = full_name.substr(dot_pos + 1); 312 std::string pre_test_name( 313 test_case_name + "." + kPreTestPrefix + test_name); 314 if (!ContainsKey(test_names_set, pre_test_name)) 315 tests_to_run_now.push_back(full_name); 316 } 317 318 for (size_t i = 0; i < tests_to_run_now.size(); i++) 319 DoRunTest(test_launcher, tests_to_run_now[i]); 320 321 return test_names_set.size(); 322} 323 324void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher, 325 const std::string& test_name) { 326 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 327 328 CommandLine cmd_line(*CommandLine::ForCurrentProcess()); 329 CHECK(launcher_delegate_->AdjustChildProcessCommandLine( 330 &cmd_line, user_data_dir_map_[test_name_no_pre])); 331 332 CommandLine new_cmd_line(cmd_line.GetProgram()); 333 CommandLine::SwitchMap switches = cmd_line.GetSwitches(); 334 335 // Strip out gtest_output flag because otherwise we would overwrite results 336 // of the other tests. 337 switches.erase(base::kGTestOutputFlag); 338 339 for (CommandLine::SwitchMap::const_iterator iter = switches.begin(); 340 iter != switches.end(); ++iter) { 341 new_cmd_line.AppendSwitchNative(iter->first, iter->second); 342 } 343 344 // Always enable disabled tests. This method is not called with disabled 345 // tests unless this flag was specified to the browser test executable. 346 new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests"); 347 new_cmd_line.AppendSwitchASCII("gtest_filter", test_name); 348 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 349 350 char* browser_wrapper = getenv("BROWSER_WRAPPER"); 351 352 parallel_launcher_.LaunchChildGTestProcess( 353 new_cmd_line, 354 browser_wrapper ? browser_wrapper : std::string(), 355 TestTimeouts::action_max_timeout(), 356 base::Bind(&WrapperTestLauncherDelegate::GTestCallback, 357 base::Unretained(this), 358 test_launcher, 359 test_name)); 360} 361 362void WrapperTestLauncherDelegate::RunDependentTest( 363 base::TestLauncher* test_launcher, 364 const std::string test_name, 365 const base::TestResult& pre_test_result) { 366 if (pre_test_result.status == base::TestResult::TEST_SUCCESS) { 367 // Only run the dependent test if PRE_ test succeeded. 368 DoRunTest(test_launcher, test_name); 369 } else { 370 // Otherwise skip the test. 371 base::TestResult test_result; 372 test_result.full_name = test_name; 373 test_result.status = base::TestResult::TEST_SKIPPED; 374 test_launcher->OnTestFinished(test_result); 375 376 if (ContainsKey(dependent_test_map_, test_name)) { 377 RunDependentTest(test_launcher, 378 dependent_test_map_[test_name], 379 test_result); 380 } 381 } 382} 383 384void WrapperTestLauncherDelegate::GTestCallback( 385 base::TestLauncher* test_launcher, 386 const std::string& test_name, 387 int exit_code, 388 const base::TimeDelta& elapsed_time, 389 bool was_timeout, 390 const std::string& output) { 391 base::TestResult result; 392 result.full_name = test_name; 393 394 // TODO(phajdan.jr): Recognize crashes. 395 if (exit_code == 0) 396 result.status = base::TestResult::TEST_SUCCESS; 397 else if (was_timeout) 398 result.status = base::TestResult::TEST_TIMEOUT; 399 else 400 result.status = base::TestResult::TEST_FAILURE; 401 402 result.elapsed_time = elapsed_time; 403 404 result.output_snippet = GetTestOutputSnippet(result, output); 405 406 if (ContainsKey(dependent_test_map_, test_name)) { 407 RunDependentTest(test_launcher, dependent_test_map_[test_name], result); 408 } else { 409 // No other tests depend on this, we can delete the temporary directory now. 410 // Do so to avoid too many temporary files using lots of disk space. 411 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 412 if (ContainsKey(user_data_dir_map_, test_name_no_pre)) { 413 if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) { 414 LOG(WARNING) << "Failed to delete " 415 << user_data_dir_map_[test_name_no_pre].value(); 416 } 417 user_data_dir_map_.erase(test_name_no_pre); 418 } 419 } 420 421 test_launcher->OnTestFinished(result); 422 parallel_launcher_.ResetOutputWatchdog(); 423} 424 425bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 426 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 427 return true; 428 429 std::string switch_value = 430 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 431 if (!base::StringToInt(switch_value, result) || *result < 1) { 432 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 433 return false; 434 } 435 436 return true; 437} 438 439} // namespace 440 441const char kHelpFlag[] = "help"; 442 443const char kLaunchAsBrowser[] = "as-browser"; 444 445// See kManualTestPrefix above. 446const char kRunManualTestsFlag[] = "run-manual"; 447 448const char kSingleProcessTestsFlag[] = "single_process"; 449 450 451TestLauncherDelegate::~TestLauncherDelegate() { 452} 453 454bool ShouldRunContentMain() { 455#if defined(OS_WIN) || defined(OS_LINUX) 456 CommandLine* command_line = CommandLine::ForCurrentProcess(); 457 return command_line->HasSwitch(switches::kProcessType) || 458 command_line->HasSwitch(kLaunchAsBrowser); 459#else 460 return false; 461#endif // defined(OS_WIN) || defined(OS_LINUX) 462} 463 464int RunContentMain(int argc, char** argv, 465 TestLauncherDelegate* launcher_delegate) { 466#if defined(OS_WIN) 467 sandbox::SandboxInterfaceInfo sandbox_info = {0}; 468 InitializeSandboxInfo(&sandbox_info); 469 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 470 launcher_delegate->CreateContentMainDelegate()); 471 return ContentMain(GetModuleHandle(NULL), 472 &sandbox_info, 473 chrome_main_delegate.get()); 474#elif defined(OS_LINUX) 475 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 476 launcher_delegate->CreateContentMainDelegate()); 477 return ContentMain(argc, const_cast<const char**>(argv), 478 chrome_main_delegate.get()); 479#endif // defined(OS_WIN) 480 NOTREACHED(); 481 return 0; 482} 483 484int LaunchTests(TestLauncherDelegate* launcher_delegate, 485 int default_jobs, 486 int argc, 487 char** argv) { 488 DCHECK(!g_launcher_delegate); 489 g_launcher_delegate = launcher_delegate; 490 491 CommandLine::Init(argc, argv); 492 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 493 494 if (command_line->HasSwitch(kHelpFlag)) { 495 PrintUsage(); 496 return 0; 497 } 498 499 if (command_line->HasSwitch(kSingleProcessTestsFlag) || 500 (command_line->HasSwitch(switches::kSingleProcess) && 501 command_line->HasSwitch(base::kGTestFilterFlag)) || 502 command_line->HasSwitch(base::kGTestListTestsFlag) || 503 command_line->HasSwitch(base::kGTestHelpFlag)) { 504#if defined(OS_WIN) 505 if (command_line->HasSwitch(kSingleProcessTestsFlag)) { 506 sandbox::SandboxInterfaceInfo sandbox_info; 507 InitializeSandboxInfo(&sandbox_info); 508 InitializeSandbox(&sandbox_info); 509 } 510#endif 511 return launcher_delegate->RunTestSuite(argc, argv); 512 } 513 514 if (ShouldRunContentMain()) 515 return RunContentMain(argc, argv, launcher_delegate); 516 517 base::AtExitManager at_exit; 518 testing::InitGoogleTest(&argc, argv); 519 TestTimeouts::Initialize(); 520 521 int jobs = default_jobs; 522 if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs)) 523 return 1; 524 525 fprintf(stdout, 526 "Starting tests (using %d parallel jobs)...\n" 527 "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n" 528 "For debugging a test inside a debugger, use the\n" 529 "--gtest_filter=<your_test_name> flag along with either\n" 530 "--single_process (to run the test in one launcher/browser process) or\n" 531 "--single-process (to do the above, and also run Chrome in single-" 532 "process mode).\n", jobs); 533 534 base::MessageLoopForIO message_loop; 535 536 WrapperTestLauncherDelegate delegate(launcher_delegate, jobs); 537 base::TestLauncher launcher(&delegate); 538 bool success = launcher.Run(argc, argv); 539 return (success ? 0 : 1); 540} 541 542TestLauncherDelegate* GetCurrentTestLauncherDelegate() { 543 return g_launcher_delegate; 544} 545 546} // namespace content 547