test_launcher.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/public/test/test_launcher.h" 6 7#include <map> 8#include <string> 9#include <vector> 10 11#include "base/command_line.h" 12#include "base/containers/hash_tables.h" 13#include "base/environment.h" 14#include "base/file_util.h" 15#include "base/files/scoped_temp_dir.h" 16#include "base/logging.h" 17#include "base/memory/linked_ptr.h" 18#include "base/memory/scoped_ptr.h" 19#include "base/message_loop/message_loop.h" 20#include "base/stl_util.h" 21#include "base/strings/string_number_conversions.h" 22#include "base/strings/string_util.h" 23#include "base/strings/utf_string_conversions.h" 24#include "base/test/launcher/test_launcher.h" 25#include "base/test/test_suite.h" 26#include "base/test/test_switches.h" 27#include "base/test/test_timeouts.h" 28#include "base/time/time.h" 29#include "content/public/app/content_main.h" 30#include "content/public/app/content_main_delegate.h" 31#include "content/public/app/startup_helper_win.h" 32#include "content/public/common/content_switches.h" 33#include "content/public/common/sandbox_init.h" 34#include "content/public/test/browser_test.h" 35#include "net/base/escape.h" 36#include "testing/gtest/include/gtest/gtest.h" 37 38#if defined(OS_WIN) 39#include "base/base_switches.h" 40#include "content/common/sandbox_win.h" 41#include "sandbox/win/src/sandbox_factory.h" 42#include "sandbox/win/src/sandbox_types.h" 43#elif defined(OS_MACOSX) 44#include "base/mac/scoped_nsautorelease_pool.h" 45#endif 46 47namespace content { 48 49namespace { 50 51// Tests with this prefix run before the same test without it, and use the same 52// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests 53// that span browser restarts. 54const char kPreTestPrefix[] = "PRE_"; 55 56// Manual tests only run when --run-manual is specified. This allows writing 57// tests that don't run automatically but are still in the same test binary. 58// This is useful so that a team that wants to run a few tests doesn't have to 59// add a new binary that must be compiled on all builds. 60const char kManualTestPrefix[] = "MANUAL_"; 61 62TestLauncherDelegate* g_launcher_delegate; 63 64std::string RemoveAnyPrePrefixes(const std::string& test_name) { 65 std::string result(test_name); 66 ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string()); 67 return result; 68} 69 70void PrintUsage() { 71 fprintf(stdout, 72 "Runs tests using the gtest framework, each batch of tests being\n" 73 "run in their own process. Supported command-line flags:\n" 74 "\n" 75 " Common flags:\n" 76 " --gtest_filter=...\n" 77 " Runs a subset of tests (see --gtest_help for more info).\n" 78 "\n" 79 " --help\n" 80 " Shows this message.\n" 81 "\n" 82 " --gtest_help\n" 83 " Shows the gtest help message.\n" 84 "\n" 85 " --test-launcher-jobs=N\n" 86 " Sets the number of parallel test jobs to N.\n" 87 "\n" 88 " --single_process\n" 89 " Runs the tests and the launcher in the same process. Useful\n" 90 " for debugging a specific test in a debugger.\n" 91 "\n" 92 " Other flags:\n" 93 " --test-launcher-retry-limit=N\n" 94 " Sets the limit of test retries on failures to N.\n" 95 "\n" 96 " --test-launcher-summary-output=PATH\n" 97 " Saves a JSON machine-readable summary of the run.\n" 98 "\n" 99 " --test-launcher-print-test-stdio=auto|always|never\n" 100 " Controls when full test output is printed.\n" 101 " auto means to print it when the test failed.\n" 102 "\n" 103 " --test-launcher-total-shards=N\n" 104 " Sets the total number of shards to N.\n" 105 "\n" 106 " --test-launcher-shard-index=N\n" 107 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"); 108} 109 110// Implementation of base::TestLauncherDelegate. This is also a test launcher, 111// wrapping a lower-level test launcher with content-specific code. 112class WrapperTestLauncherDelegate : public base::TestLauncherDelegate { 113 public: 114 explicit WrapperTestLauncherDelegate( 115 content::TestLauncherDelegate* launcher_delegate) 116 : launcher_delegate_(launcher_delegate) { 117 CHECK(temp_dir_.CreateUniqueTempDir()); 118 } 119 120 // base::TestLauncherDelegate: 121 virtual bool ShouldRunTest(const testing::TestCase* test_case, 122 const testing::TestInfo* test_info) OVERRIDE; 123 virtual size_t RunTests(base::TestLauncher* test_launcher, 124 const std::vector<std::string>& test_names) OVERRIDE; 125 virtual size_t RetryTests( 126 base::TestLauncher* test_launcher, 127 const std::vector<std::string>& test_names) OVERRIDE; 128 129 private: 130 void DoRunTest(base::TestLauncher* test_launcher, 131 const std::string& test_name); 132 133 // Launches test named |test_name| using parallel launcher, 134 // given result of PRE_ test |pre_test_result|. 135 void RunDependentTest(base::TestLauncher* test_launcher, 136 const std::string test_name, 137 const base::TestResult& pre_test_result); 138 139 // Callback to receive result of a test. 140 void GTestCallback( 141 base::TestLauncher* test_launcher, 142 const std::string& test_name, 143 int exit_code, 144 const base::TimeDelta& elapsed_time, 145 bool was_timeout, 146 const std::string& output); 147 148 content::TestLauncherDelegate* launcher_delegate_; 149 150 // Store dependent test name (map is indexed by full test name). 151 typedef std::map<std::string, std::string> DependentTestMap; 152 DependentTestMap dependent_test_map_; 153 DependentTestMap reverse_dependent_test_map_; 154 155 // Store unique data directory prefix for test names (without PRE_ prefixes). 156 // PRE_ tests and tests that depend on them must share the same 157 // data directory. Using test name as directory name leads to too long 158 // names (exceeding UNIX_PATH_MAX, which creates a problem with 159 // process_singleton_linux). Create a randomly-named temporary directory 160 // and keep track of the names so that PRE_ tests can still re-use them. 161 typedef std::map<std::string, base::FilePath> UserDataDirMap; 162 UserDataDirMap user_data_dir_map_; 163 164 // Store names of all seen tests to properly handle PRE_ tests. 165 std::set<std::string> all_test_names_; 166 167 // Temporary directory for user data directories. 168 base::ScopedTempDir temp_dir_; 169 170 DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate); 171}; 172 173bool WrapperTestLauncherDelegate::ShouldRunTest( 174 const testing::TestCase* test_case, 175 const testing::TestInfo* test_info) { 176 all_test_names_.insert( 177 std::string(test_case->name()) + "." + test_info->name()); 178 179 if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) && 180 !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) { 181 return false; 182 } 183 184 if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) { 185 // We will actually run PRE_ tests, but to ensure they run on the same shard 186 // as dependent tests, handle all these details internally. 187 return false; 188 } 189 190 return true; 191} 192 193std::string GetPreTestName(const std::string& full_name) { 194 size_t dot_pos = full_name.find('.'); 195 CHECK_NE(dot_pos, std::string::npos); 196 std::string test_case_name = full_name.substr(0, dot_pos); 197 std::string test_name = full_name.substr(dot_pos + 1); 198 return test_case_name + "." + kPreTestPrefix + test_name; 199} 200 201size_t WrapperTestLauncherDelegate::RunTests( 202 base::TestLauncher* test_launcher, 203 const std::vector<std::string>& test_names) { 204 dependent_test_map_.clear(); 205 reverse_dependent_test_map_.clear(); 206 user_data_dir_map_.clear(); 207 208 // Number of additional tests to run because of dependencies. 209 size_t additional_tests_to_run_count = 0; 210 211 // Compute dependencies of tests to be run. 212 for (size_t i = 0; i < test_names.size(); i++) { 213 std::string full_name(test_names[i]); 214 std::string pre_test_name(GetPreTestName(full_name)); 215 216 while (ContainsKey(all_test_names_, pre_test_name)) { 217 additional_tests_to_run_count++; 218 219 DCHECK(!ContainsKey(dependent_test_map_, pre_test_name)); 220 dependent_test_map_[pre_test_name] = full_name; 221 222 DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name)); 223 reverse_dependent_test_map_[full_name] = pre_test_name; 224 225 full_name = pre_test_name; 226 pre_test_name = GetPreTestName(pre_test_name); 227 } 228 } 229 230 for (size_t i = 0; i < test_names.size(); i++) { 231 std::string full_name(test_names[i]); 232 233 // Make sure no PRE_ tests were requested explicitly. 234 DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name)); 235 236 if (!ContainsKey(user_data_dir_map_, full_name)) { 237 base::FilePath temp_dir; 238 CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(), 239 FILE_PATH_LITERAL("d"), &temp_dir)); 240 user_data_dir_map_[full_name] = temp_dir; 241 } 242 243 // If the test has any dependencies, get to the root and start with that. 244 while (ContainsKey(reverse_dependent_test_map_, full_name)) 245 full_name = GetPreTestName(full_name); 246 247 DoRunTest(test_launcher, full_name); 248 } 249 250 return test_names.size() + additional_tests_to_run_count; 251} 252 253size_t WrapperTestLauncherDelegate::RetryTests( 254 base::TestLauncher* test_launcher, 255 const std::vector<std::string>& test_names) { 256 // List of tests we can kick off right now, depending on no other tests. 257 std::vector<std::string> tests_to_run_now; 258 259 // We retry at least the tests requested to retry. 260 std::set<std::string> test_names_set(test_names.begin(), test_names.end()); 261 262 // In the face of PRE_ tests, we need to retry the entire chain of tests, 263 // from the very first one. 264 for (size_t i = 0; i < test_names.size(); i++) { 265 std::string test_name(test_names[i]); 266 while (ContainsKey(reverse_dependent_test_map_, test_name)) { 267 test_name = reverse_dependent_test_map_[test_name]; 268 test_names_set.insert(test_name); 269 } 270 } 271 272 // Discard user data directories from any previous runs. Start with 273 // fresh state. 274 for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin(); 275 i != user_data_dir_map_.end(); 276 ++i) { 277 // Delete temporary directories now to avoid using too much space in /tmp. 278 if (!base::DeleteFile(i->second, true)) { 279 LOG(WARNING) << "Failed to delete " << i->second.value(); 280 } 281 } 282 user_data_dir_map_.clear(); 283 284 for (std::set<std::string>::const_iterator i = test_names_set.begin(); 285 i != test_names_set.end(); 286 ++i) { 287 std::string full_name(*i); 288 289 // Make sure PRE_ tests and tests that depend on them share the same 290 // data directory - based it on the test name without prefixes. 291 std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name)); 292 if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) { 293 base::FilePath temp_dir; 294 CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(), 295 FILE_PATH_LITERAL("d"), &temp_dir)); 296 user_data_dir_map_[test_name_no_pre] = temp_dir; 297 } 298 299 size_t dot_pos = full_name.find('.'); 300 CHECK_NE(dot_pos, std::string::npos); 301 std::string test_case_name = full_name.substr(0, dot_pos); 302 std::string test_name = full_name.substr(dot_pos + 1); 303 std::string pre_test_name( 304 test_case_name + "." + kPreTestPrefix + test_name); 305 if (!ContainsKey(test_names_set, pre_test_name)) 306 tests_to_run_now.push_back(full_name); 307 } 308 309 for (size_t i = 0; i < tests_to_run_now.size(); i++) 310 DoRunTest(test_launcher, tests_to_run_now[i]); 311 312 return test_names_set.size(); 313} 314 315void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher, 316 const std::string& test_name) { 317 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 318 319 CommandLine cmd_line(*CommandLine::ForCurrentProcess()); 320 CHECK(launcher_delegate_->AdjustChildProcessCommandLine( 321 &cmd_line, user_data_dir_map_[test_name_no_pre])); 322 323 CommandLine new_cmd_line(cmd_line.GetProgram()); 324 CommandLine::SwitchMap switches = cmd_line.GetSwitches(); 325 326 // Strip out gtest_output flag because otherwise we would overwrite results 327 // of the other tests. 328 switches.erase(base::kGTestOutputFlag); 329 330 for (CommandLine::SwitchMap::const_iterator iter = switches.begin(); 331 iter != switches.end(); ++iter) { 332 new_cmd_line.AppendSwitchNative(iter->first, iter->second); 333 } 334 335 // Always enable disabled tests. This method is not called with disabled 336 // tests unless this flag was specified to the browser test executable. 337 new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests"); 338 new_cmd_line.AppendSwitchASCII("gtest_filter", test_name); 339 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 340 341 char* browser_wrapper = getenv("BROWSER_WRAPPER"); 342 343 test_launcher->LaunchChildGTestProcess( 344 new_cmd_line, 345 browser_wrapper ? browser_wrapper : std::string(), 346 TestTimeouts::action_max_timeout(), 347 base::Bind(&WrapperTestLauncherDelegate::GTestCallback, 348 base::Unretained(this), 349 test_launcher, 350 test_name)); 351} 352 353void WrapperTestLauncherDelegate::RunDependentTest( 354 base::TestLauncher* test_launcher, 355 const std::string test_name, 356 const base::TestResult& pre_test_result) { 357 if (pre_test_result.status == base::TestResult::TEST_SUCCESS) { 358 // Only run the dependent test if PRE_ test succeeded. 359 DoRunTest(test_launcher, test_name); 360 } else { 361 // Otherwise skip the test. 362 base::TestResult test_result; 363 test_result.full_name = test_name; 364 test_result.status = base::TestResult::TEST_SKIPPED; 365 test_launcher->OnTestFinished(test_result); 366 367 if (ContainsKey(dependent_test_map_, test_name)) { 368 RunDependentTest(test_launcher, 369 dependent_test_map_[test_name], 370 test_result); 371 } 372 } 373} 374 375void WrapperTestLauncherDelegate::GTestCallback( 376 base::TestLauncher* test_launcher, 377 const std::string& test_name, 378 int exit_code, 379 const base::TimeDelta& elapsed_time, 380 bool was_timeout, 381 const std::string& output) { 382 base::TestResult result; 383 result.full_name = test_name; 384 385 // TODO(phajdan.jr): Recognize crashes. 386 if (exit_code == 0) 387 result.status = base::TestResult::TEST_SUCCESS; 388 else if (was_timeout) 389 result.status = base::TestResult::TEST_TIMEOUT; 390 else 391 result.status = base::TestResult::TEST_FAILURE; 392 393 result.elapsed_time = elapsed_time; 394 395 result.output_snippet = GetTestOutputSnippet(result, output); 396 397 if (ContainsKey(dependent_test_map_, test_name)) { 398 RunDependentTest(test_launcher, dependent_test_map_[test_name], result); 399 } else { 400 // No other tests depend on this, we can delete the temporary directory now. 401 // Do so to avoid too many temporary files using lots of disk space. 402 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 403 if (ContainsKey(user_data_dir_map_, test_name_no_pre)) { 404 if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) { 405 LOG(WARNING) << "Failed to delete " 406 << user_data_dir_map_[test_name_no_pre].value(); 407 } 408 user_data_dir_map_.erase(test_name_no_pre); 409 } 410 } 411 412 test_launcher->OnTestFinished(result); 413} 414 415} // namespace 416 417const char kHelpFlag[] = "help"; 418 419const char kLaunchAsBrowser[] = "as-browser"; 420 421// See kManualTestPrefix above. 422const char kRunManualTestsFlag[] = "run-manual"; 423 424const char kSingleProcessTestsFlag[] = "single_process"; 425 426 427TestLauncherDelegate::~TestLauncherDelegate() { 428} 429 430#if defined(OS_WIN) || defined(OS_LINUX) 431bool ShouldRunContentMain() { 432 CommandLine* command_line = CommandLine::ForCurrentProcess(); 433 return command_line->HasSwitch(switches::kProcessType) || 434 command_line->HasSwitch(kLaunchAsBrowser); 435 return false; 436} 437 438int RunContentMain(int argc, char** argv, 439 TestLauncherDelegate* launcher_delegate) { 440 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 441 launcher_delegate->CreateContentMainDelegate()); 442 ContentMainParams params(chrome_main_delegate.get()); 443 444#if defined(OS_WIN) 445 sandbox::SandboxInterfaceInfo sandbox_info = {0}; 446 InitializeSandboxInfo(&sandbox_info); 447 448 params.instance = GetModuleHandle(NULL); 449 params.sandbox_info = &sandbox_info; 450#elif defined(OS_LINUX) 451 params.argc = argc; 452 params.argv = const_cast<const char**>(argv); 453#endif // defined(OS_WIN) 454 455 return ContentMain(params); 456} 457#endif 458 459int LaunchTests(TestLauncherDelegate* launcher_delegate, 460 int default_jobs, 461 int argc, 462 char** argv) { 463 DCHECK(!g_launcher_delegate); 464 g_launcher_delegate = launcher_delegate; 465 466 CommandLine::Init(argc, argv); 467 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 468 469 if (command_line->HasSwitch(kHelpFlag)) { 470 PrintUsage(); 471 return 0; 472 } 473 474 if (command_line->HasSwitch(kSingleProcessTestsFlag) || 475 (command_line->HasSwitch(switches::kSingleProcess) && 476 command_line->HasSwitch(base::kGTestFilterFlag)) || 477 command_line->HasSwitch(base::kGTestListTestsFlag) || 478 command_line->HasSwitch(base::kGTestHelpFlag)) { 479#if defined(OS_WIN) 480 if (command_line->HasSwitch(kSingleProcessTestsFlag)) { 481 sandbox::SandboxInterfaceInfo sandbox_info; 482 InitializeSandboxInfo(&sandbox_info); 483 InitializeSandbox(&sandbox_info); 484 } 485#endif 486 return launcher_delegate->RunTestSuite(argc, argv); 487 } 488 489#if defined(OS_WIN) || defined(OS_LINUX) 490 if (ShouldRunContentMain()) 491 return RunContentMain(argc, argv, launcher_delegate); 492#endif 493 494 base::AtExitManager at_exit; 495 testing::InitGoogleTest(&argc, argv); 496 TestTimeouts::Initialize(); 497 498 fprintf(stdout, 499 "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n" 500 "For debugging a test inside a debugger, use the\n" 501 "--gtest_filter=<your_test_name> flag along with either\n" 502 "--single_process (to run the test in one launcher/browser process) or\n" 503 "--single-process (to do the above, and also run Chrome in single-" 504 "process mode).\n"); 505 506 base::MessageLoopForIO message_loop; 507 508 // Allow the |launcher_delegate| to modify |default_jobs|. 509 launcher_delegate->AdjustDefaultParallelJobs(&default_jobs); 510 511 WrapperTestLauncherDelegate delegate(launcher_delegate); 512 base::TestLauncher launcher(&delegate, default_jobs); 513 bool success = launcher.Run(argc, argv); 514 return (success ? 0 : 1); 515} 516 517TestLauncherDelegate* GetCurrentTestLauncherDelegate() { 518 return g_launcher_delegate; 519} 520 521} // namespace content 522