test_launcher.cc revision f2477e01787aa58f445919b809d89e252beef54f
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/public/test/test_launcher.h" 6 7#include <map> 8#include <string> 9#include <vector> 10 11#include "base/command_line.h" 12#include "base/containers/hash_tables.h" 13#include "base/environment.h" 14#include "base/file_util.h" 15#include "base/files/scoped_temp_dir.h" 16#include "base/logging.h" 17#include "base/memory/linked_ptr.h" 18#include "base/memory/scoped_ptr.h" 19#include "base/message_loop/message_loop.h" 20#include "base/stl_util.h" 21#include "base/strings/string_number_conversions.h" 22#include "base/strings/string_util.h" 23#include "base/strings/utf_string_conversions.h" 24#include "base/test/launcher/test_launcher.h" 25#include "base/test/test_suite.h" 26#include "base/test/test_switches.h" 27#include "base/test/test_timeouts.h" 28#include "base/time/time.h" 29#include "content/public/app/content_main.h" 30#include "content/public/app/content_main_delegate.h" 31#include "content/public/app/startup_helper_win.h" 32#include "content/public/common/content_switches.h" 33#include "content/public/common/sandbox_init.h" 34#include "content/public/test/browser_test.h" 35#include "net/base/escape.h" 36#include "testing/gtest/include/gtest/gtest.h" 37 38#if defined(OS_WIN) 39#include "base/base_switches.h" 40#include "content/common/sandbox_win.h" 41#include "sandbox/win/src/sandbox_factory.h" 42#include "sandbox/win/src/sandbox_types.h" 43#elif defined(OS_MACOSX) 44#include "base/mac/scoped_nsautorelease_pool.h" 45#endif 46 47namespace content { 48 49namespace { 50 51// Tests with this prefix run before the same test without it, and use the same 52// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests 53// that span browser restarts. 54const char kPreTestPrefix[] = "PRE_"; 55 56// Manual tests only run when --run-manual is specified. This allows writing 57// tests that don't run automatically but are still in the same test binary. 58// This is useful so that a team that wants to run a few tests doesn't have to 59// add a new binary that must be compiled on all builds. 60const char kManualTestPrefix[] = "MANUAL_"; 61 62TestLauncherDelegate* g_launcher_delegate; 63 64std::string RemoveAnyPrePrefixes(const std::string& test_name) { 65 std::string result(test_name); 66 ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string()); 67 return result; 68} 69 70void PrintUsage() { 71 fprintf(stdout, 72 "Runs tests using the gtest framework, each test being run in its own\n" 73 "process. Any gtest flags can be specified.\n" 74 " --single_process\n" 75 " Runs the tests and the launcher in the same process. Useful for \n" 76 " debugging a specific test in a debugger.\n" 77 " --single-process\n" 78 " Same as above, and also runs Chrome in single-process mode.\n" 79 " --help\n" 80 " Shows this message.\n" 81 " --gtest_help\n" 82 " Shows the gtest help message.\n"); 83} 84 85// Implementation of base::TestLauncherDelegate. This is also a test launcher, 86// wrapping a lower-level test launcher with content-specific code. 87class WrapperTestLauncherDelegate : public base::TestLauncherDelegate { 88 public: 89 explicit WrapperTestLauncherDelegate( 90 content::TestLauncherDelegate* launcher_delegate) 91 : launcher_delegate_(launcher_delegate) { 92 CHECK(temp_dir_.CreateUniqueTempDir()); 93 } 94 95 // base::TestLauncherDelegate: 96 virtual void OnTestIterationStarting() OVERRIDE; 97 virtual std::string GetTestNameForFiltering( 98 const testing::TestCase* test_case, 99 const testing::TestInfo* test_info) OVERRIDE; 100 virtual bool ShouldRunTest(const testing::TestCase* test_case, 101 const testing::TestInfo* test_info) OVERRIDE; 102 virtual size_t RunTests(base::TestLauncher* test_launcher, 103 const std::vector<std::string>& test_names) OVERRIDE; 104 virtual size_t RetryTests( 105 base::TestLauncher* test_launcher, 106 const std::vector<std::string>& test_names) OVERRIDE; 107 108 private: 109 void DoRunTest(base::TestLauncher* test_launcher, 110 const std::string& test_name); 111 112 // Launches test named |test_name| using parallel launcher, 113 // given result of PRE_ test |pre_test_result|. 114 void RunDependentTest(base::TestLauncher* test_launcher, 115 const std::string test_name, 116 const base::TestResult& pre_test_result); 117 118 // Callback to receive result of a test. 119 void GTestCallback( 120 base::TestLauncher* test_launcher, 121 const std::string& test_name, 122 int exit_code, 123 const base::TimeDelta& elapsed_time, 124 bool was_timeout, 125 const std::string& output); 126 127 content::TestLauncherDelegate* launcher_delegate_; 128 129 // Store dependent test name (map is indexed by full test name). 130 typedef std::map<std::string, std::string> DependentTestMap; 131 DependentTestMap dependent_test_map_; 132 DependentTestMap reverse_dependent_test_map_; 133 134 // Store unique data directory prefix for test names (without PRE_ prefixes). 135 // PRE_ tests and tests that depend on them must share the same 136 // data directory. Using test name as directory name leads to too long 137 // names (exceeding UNIX_PATH_MAX, which creates a problem with 138 // process_singleton_linux). Create a randomly-named temporary directory 139 // and keep track of the names so that PRE_ tests can still re-use them. 140 typedef std::map<std::string, base::FilePath> UserDataDirMap; 141 UserDataDirMap user_data_dir_map_; 142 143 // Store names of all seen tests to properly handle PRE_ tests. 144 std::set<std::string> all_test_names_; 145 146 // Temporary directory for user data directories. 147 base::ScopedTempDir temp_dir_; 148 149 DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate); 150}; 151 152void WrapperTestLauncherDelegate::OnTestIterationStarting() { 153 dependent_test_map_.clear(); 154 user_data_dir_map_.clear(); 155} 156 157std::string WrapperTestLauncherDelegate::GetTestNameForFiltering( 158 const testing::TestCase* test_case, 159 const testing::TestInfo* test_info) { 160 return RemoveAnyPrePrefixes( 161 std::string(test_case->name()) + "." + test_info->name()); 162} 163 164bool WrapperTestLauncherDelegate::ShouldRunTest( 165 const testing::TestCase* test_case, 166 const testing::TestInfo* test_info) { 167 all_test_names_.insert( 168 std::string(test_case->name()) + "." + test_info->name()); 169 170 if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) && 171 !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) { 172 return false; 173 } 174 175 if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) { 176 // We will actually run PRE_ tests, but to ensure they run on the same shard 177 // as dependent tests, handle all these details internally. 178 return false; 179 } 180 181 return true; 182} 183 184std::string GetPreTestName(const std::string& full_name) { 185 size_t dot_pos = full_name.find('.'); 186 CHECK_NE(dot_pos, std::string::npos); 187 std::string test_case_name = full_name.substr(0, dot_pos); 188 std::string test_name = full_name.substr(dot_pos + 1); 189 return test_case_name + "." + kPreTestPrefix + test_name; 190} 191 192size_t WrapperTestLauncherDelegate::RunTests( 193 base::TestLauncher* test_launcher, 194 const std::vector<std::string>& test_names) { 195 // Number of additional tests to run because of dependencies. 196 size_t additional_tests_to_run_count = 0; 197 198 // Compute dependencies of tests to be run. 199 for (size_t i = 0; i < test_names.size(); i++) { 200 std::string full_name(test_names[i]); 201 std::string pre_test_name(GetPreTestName(full_name)); 202 203 while (ContainsKey(all_test_names_, pre_test_name)) { 204 additional_tests_to_run_count++; 205 206 DCHECK(!ContainsKey(dependent_test_map_, pre_test_name)); 207 dependent_test_map_[pre_test_name] = full_name; 208 209 DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name)); 210 reverse_dependent_test_map_[full_name] = pre_test_name; 211 212 full_name = pre_test_name; 213 pre_test_name = GetPreTestName(pre_test_name); 214 } 215 } 216 217 for (size_t i = 0; i < test_names.size(); i++) { 218 std::string full_name(test_names[i]); 219 220 // Make sure no PRE_ tests were requested explicitly. 221 DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name)); 222 223 if (!ContainsKey(user_data_dir_map_, full_name)) { 224 base::FilePath temp_dir; 225 CHECK(file_util::CreateTemporaryDirInDir( 226 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir)); 227 user_data_dir_map_[full_name] = temp_dir; 228 } 229 230 // If the test has any dependencies, get to the root and start with that. 231 while (ContainsKey(reverse_dependent_test_map_, full_name)) 232 full_name = GetPreTestName(full_name); 233 234 DoRunTest(test_launcher, full_name); 235 } 236 237 return test_names.size() + additional_tests_to_run_count; 238} 239 240size_t WrapperTestLauncherDelegate::RetryTests( 241 base::TestLauncher* test_launcher, 242 const std::vector<std::string>& test_names) { 243 // List of tests we can kick off right now, depending on no other tests. 244 std::vector<std::string> tests_to_run_now; 245 246 // We retry at least the tests requested to retry. 247 std::set<std::string> test_names_set(test_names.begin(), test_names.end()); 248 249 // In the face of PRE_ tests, we need to retry the entire chain of tests, 250 // from the very first one. 251 for (size_t i = 0; i < test_names.size(); i++) { 252 std::string test_name(test_names[i]); 253 while (ContainsKey(reverse_dependent_test_map_, test_name)) { 254 test_name = reverse_dependent_test_map_[test_name]; 255 test_names_set.insert(test_name); 256 } 257 } 258 259 // Discard user data directories from any previous runs. Start with 260 // fresh state. 261 for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin(); 262 i != user_data_dir_map_.end(); 263 ++i) { 264 // Delete temporary directories now to avoid using too much space in /tmp. 265 if (!base::DeleteFile(i->second, true)) { 266 LOG(WARNING) << "Failed to delete " << i->second.value(); 267 } 268 } 269 user_data_dir_map_.clear(); 270 271 for (std::set<std::string>::const_iterator i = test_names_set.begin(); 272 i != test_names_set.end(); 273 ++i) { 274 std::string full_name(*i); 275 276 // Make sure PRE_ tests and tests that depend on them share the same 277 // data directory - based it on the test name without prefixes. 278 std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name)); 279 if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) { 280 base::FilePath temp_dir; 281 CHECK(file_util::CreateTemporaryDirInDir( 282 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir)); 283 user_data_dir_map_[test_name_no_pre] = temp_dir; 284 } 285 286 size_t dot_pos = full_name.find('.'); 287 CHECK_NE(dot_pos, std::string::npos); 288 std::string test_case_name = full_name.substr(0, dot_pos); 289 std::string test_name = full_name.substr(dot_pos + 1); 290 std::string pre_test_name( 291 test_case_name + "." + kPreTestPrefix + test_name); 292 if (!ContainsKey(test_names_set, pre_test_name)) 293 tests_to_run_now.push_back(full_name); 294 } 295 296 for (size_t i = 0; i < tests_to_run_now.size(); i++) 297 DoRunTest(test_launcher, tests_to_run_now[i]); 298 299 return test_names_set.size(); 300} 301 302void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher, 303 const std::string& test_name) { 304 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 305 306 CommandLine cmd_line(*CommandLine::ForCurrentProcess()); 307 CHECK(launcher_delegate_->AdjustChildProcessCommandLine( 308 &cmd_line, user_data_dir_map_[test_name_no_pre])); 309 310 CommandLine new_cmd_line(cmd_line.GetProgram()); 311 CommandLine::SwitchMap switches = cmd_line.GetSwitches(); 312 313 // Strip out gtest_output flag because otherwise we would overwrite results 314 // of the other tests. 315 switches.erase(base::kGTestOutputFlag); 316 317 for (CommandLine::SwitchMap::const_iterator iter = switches.begin(); 318 iter != switches.end(); ++iter) { 319 new_cmd_line.AppendSwitchNative(iter->first, iter->second); 320 } 321 322 // Always enable disabled tests. This method is not called with disabled 323 // tests unless this flag was specified to the browser test executable. 324 new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests"); 325 new_cmd_line.AppendSwitchASCII("gtest_filter", test_name); 326 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 327 328 char* browser_wrapper = getenv("BROWSER_WRAPPER"); 329 330 test_launcher->LaunchChildGTestProcess( 331 new_cmd_line, 332 browser_wrapper ? browser_wrapper : std::string(), 333 TestTimeouts::action_max_timeout(), 334 base::Bind(&WrapperTestLauncherDelegate::GTestCallback, 335 base::Unretained(this), 336 test_launcher, 337 test_name)); 338} 339 340void WrapperTestLauncherDelegate::RunDependentTest( 341 base::TestLauncher* test_launcher, 342 const std::string test_name, 343 const base::TestResult& pre_test_result) { 344 if (pre_test_result.status == base::TestResult::TEST_SUCCESS) { 345 // Only run the dependent test if PRE_ test succeeded. 346 DoRunTest(test_launcher, test_name); 347 } else { 348 // Otherwise skip the test. 349 base::TestResult test_result; 350 test_result.full_name = test_name; 351 test_result.status = base::TestResult::TEST_SKIPPED; 352 test_launcher->OnTestFinished(test_result); 353 354 if (ContainsKey(dependent_test_map_, test_name)) { 355 RunDependentTest(test_launcher, 356 dependent_test_map_[test_name], 357 test_result); 358 } 359 } 360} 361 362void WrapperTestLauncherDelegate::GTestCallback( 363 base::TestLauncher* test_launcher, 364 const std::string& test_name, 365 int exit_code, 366 const base::TimeDelta& elapsed_time, 367 bool was_timeout, 368 const std::string& output) { 369 base::TestResult result; 370 result.full_name = test_name; 371 372 // TODO(phajdan.jr): Recognize crashes. 373 if (exit_code == 0) 374 result.status = base::TestResult::TEST_SUCCESS; 375 else if (was_timeout) 376 result.status = base::TestResult::TEST_TIMEOUT; 377 else 378 result.status = base::TestResult::TEST_FAILURE; 379 380 result.elapsed_time = elapsed_time; 381 382 result.output_snippet = GetTestOutputSnippet(result, output); 383 384 if (ContainsKey(dependent_test_map_, test_name)) { 385 RunDependentTest(test_launcher, dependent_test_map_[test_name], result); 386 } else { 387 // No other tests depend on this, we can delete the temporary directory now. 388 // Do so to avoid too many temporary files using lots of disk space. 389 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name)); 390 if (ContainsKey(user_data_dir_map_, test_name_no_pre)) { 391 if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) { 392 LOG(WARNING) << "Failed to delete " 393 << user_data_dir_map_[test_name_no_pre].value(); 394 } 395 user_data_dir_map_.erase(test_name_no_pre); 396 } 397 } 398 399 test_launcher->OnTestFinished(result); 400} 401 402} // namespace 403 404const char kHelpFlag[] = "help"; 405 406const char kLaunchAsBrowser[] = "as-browser"; 407 408// See kManualTestPrefix above. 409const char kRunManualTestsFlag[] = "run-manual"; 410 411const char kSingleProcessTestsFlag[] = "single_process"; 412 413 414TestLauncherDelegate::~TestLauncherDelegate() { 415} 416 417bool ShouldRunContentMain() { 418#if defined(OS_WIN) || defined(OS_LINUX) 419 CommandLine* command_line = CommandLine::ForCurrentProcess(); 420 return command_line->HasSwitch(switches::kProcessType) || 421 command_line->HasSwitch(kLaunchAsBrowser); 422#else 423 return false; 424#endif // defined(OS_WIN) || defined(OS_LINUX) 425} 426 427int RunContentMain(int argc, char** argv, 428 TestLauncherDelegate* launcher_delegate) { 429#if defined(OS_WIN) 430 sandbox::SandboxInterfaceInfo sandbox_info = {0}; 431 InitializeSandboxInfo(&sandbox_info); 432 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 433 launcher_delegate->CreateContentMainDelegate()); 434 return ContentMain(GetModuleHandle(NULL), 435 &sandbox_info, 436 chrome_main_delegate.get()); 437#elif defined(OS_LINUX) 438 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 439 launcher_delegate->CreateContentMainDelegate()); 440 return ContentMain(argc, const_cast<const char**>(argv), 441 chrome_main_delegate.get()); 442#endif // defined(OS_WIN) 443 NOTREACHED(); 444 return 0; 445} 446 447int LaunchTests(TestLauncherDelegate* launcher_delegate, 448 int default_jobs, 449 int argc, 450 char** argv) { 451 DCHECK(!g_launcher_delegate); 452 g_launcher_delegate = launcher_delegate; 453 454 CommandLine::Init(argc, argv); 455 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 456 457 if (command_line->HasSwitch(kHelpFlag)) { 458 PrintUsage(); 459 return 0; 460 } 461 462 if (command_line->HasSwitch(kSingleProcessTestsFlag) || 463 (command_line->HasSwitch(switches::kSingleProcess) && 464 command_line->HasSwitch(base::kGTestFilterFlag)) || 465 command_line->HasSwitch(base::kGTestListTestsFlag) || 466 command_line->HasSwitch(base::kGTestHelpFlag)) { 467#if defined(OS_WIN) 468 if (command_line->HasSwitch(kSingleProcessTestsFlag)) { 469 sandbox::SandboxInterfaceInfo sandbox_info; 470 InitializeSandboxInfo(&sandbox_info); 471 InitializeSandbox(&sandbox_info); 472 } 473#endif 474 return launcher_delegate->RunTestSuite(argc, argv); 475 } 476 477 if (ShouldRunContentMain()) 478 return RunContentMain(argc, argv, launcher_delegate); 479 480 base::AtExitManager at_exit; 481 testing::InitGoogleTest(&argc, argv); 482 TestTimeouts::Initialize(); 483 484 fprintf(stdout, 485 "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n" 486 "For debugging a test inside a debugger, use the\n" 487 "--gtest_filter=<your_test_name> flag along with either\n" 488 "--single_process (to run the test in one launcher/browser process) or\n" 489 "--single-process (to do the above, and also run Chrome in single-" 490 "process mode).\n"); 491 492 base::MessageLoopForIO message_loop; 493 494 WrapperTestLauncherDelegate delegate(launcher_delegate); 495 base::TestLauncher launcher(&delegate, default_jobs); 496 bool success = launcher.Run(argc, argv); 497 return (success ? 0 : 1); 498} 499 500TestLauncherDelegate* GetCurrentTestLauncherDelegate() { 501 return g_launcher_delegate; 502} 503 504} // namespace content 505