test_launcher.cc revision 4e180b6a0b4720a9b8e9e959a882386f690f08ff
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "content/public/test/test_launcher.h" 6 7#include <map> 8#include <string> 9#include <vector> 10 11#include "base/command_line.h" 12#include "base/containers/hash_tables.h" 13#include "base/environment.h" 14#include "base/file_util.h" 15#include "base/files/scoped_temp_dir.h" 16#include "base/logging.h" 17#include "base/memory/linked_ptr.h" 18#include "base/memory/scoped_ptr.h" 19#include "base/message_loop/message_loop.h" 20#include "base/stl_util.h" 21#include "base/strings/string_number_conversions.h" 22#include "base/strings/string_util.h" 23#include "base/strings/utf_string_conversions.h" 24#include "base/test/launcher/parallel_test_launcher.h" 25#include "base/test/launcher/test_launcher.h" 26#include "base/test/test_suite.h" 27#include "base/test/test_switches.h" 28#include "base/test/test_timeouts.h" 29#include "base/time/time.h" 30#include "content/public/app/content_main.h" 31#include "content/public/app/content_main_delegate.h" 32#include "content/public/app/startup_helper_win.h" 33#include "content/public/common/content_switches.h" 34#include "content/public/common/sandbox_init.h" 35#include "content/public/test/browser_test.h" 36#include "net/base/escape.h" 37#include "testing/gtest/include/gtest/gtest.h" 38 39#if defined(OS_WIN) 40#include "base/base_switches.h" 41#include "content/common/sandbox_win.h" 42#include "sandbox/win/src/sandbox_factory.h" 43#include "sandbox/win/src/sandbox_types.h" 44#elif defined(OS_MACOSX) 45#include "base/mac/scoped_nsautorelease_pool.h" 46#endif 47 48namespace content { 49 50namespace { 51 52// Tests with this prefix run before the same test without it, and use the same 53// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests 54// that span browser restarts. 55const char kPreTestPrefix[] = "PRE_"; 56 57// Manual tests only run when --run-manual is specified. This allows writing 58// tests that don't run automatically but are still in the same test binary. 59// This is useful so that a team that wants to run a few tests doesn't have to 60// add a new binary that must be compiled on all builds. 61const char kManualTestPrefix[] = "MANUAL_"; 62 63TestLauncherDelegate* g_launcher_delegate; 64 65std::string RemoveAnyPrePrefixes(const std::string& test_name) { 66 std::string result(test_name); 67 ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string()); 68 return result; 69} 70 71void PrintUsage() { 72 fprintf(stdout, 73 "Runs tests using the gtest framework, each test being run in its own\n" 74 "process. Any gtest flags can be specified.\n" 75 " --single_process\n" 76 " Runs the tests and the launcher in the same process. Useful for \n" 77 " debugging a specific test in a debugger.\n" 78 " --single-process\n" 79 " Same as above, and also runs Chrome in single-process mode.\n" 80 " --help\n" 81 " Shows this message.\n" 82 " --gtest_help\n" 83 " Shows the gtest help message.\n"); 84} 85 86// Implementation of base::TestLauncherDelegate. This is also a test launcher, 87// wrapping a lower-level test launcher with content-specific code. 88class WrapperTestLauncherDelegate : public base::TestLauncherDelegate { 89 public: 90 WrapperTestLauncherDelegate(content::TestLauncherDelegate* launcher_delegate, 91 size_t jobs) 92 : launcher_delegate_(launcher_delegate), 93 timeout_count_(0), 94 printed_timeout_message_(false), 95 parallel_launcher_(jobs) { 96 CHECK(temp_dir_.CreateUniqueTempDir()); 97 } 98 99 // base::TestLauncherDelegate: 100 virtual void OnTestIterationStarting() OVERRIDE; 101 virtual std::string GetTestNameForFiltering( 102 const testing::TestCase* test_case, 103 const testing::TestInfo* test_info) OVERRIDE; 104 virtual bool ShouldRunTest(const testing::TestCase* test_case, 105 const testing::TestInfo* test_info) OVERRIDE; 106 virtual void RunTest( 107 const testing::TestCase* test_case, 108 const testing::TestInfo* test_info, 109 const base::TestLauncherDelegate::TestResultCallback& callback) OVERRIDE; 110 virtual void RunRemainingTests() OVERRIDE; 111 112 private: 113 struct TestInfo { 114 std::string GetFullName() const { return test_case_name + "." + test_name; } 115 116 std::string test_case_name; 117 std::string test_name; 118 std::vector<base::TestLauncherDelegate::TestResultCallback> callbacks; 119 }; 120 121 // Launches test from |test_info| using |command_line| and parallel launcher. 122 void DoRunTest(const TestInfo& test_info, const CommandLine& command_line); 123 124 // Launches test named |test_name| using |command_line| and parallel launcher, 125 // given result of PRE_ test |pre_test_result|. 126 void RunDependentTest(const std::string test_name, 127 const CommandLine& command_line, 128 const base::TestResult& pre_test_result); 129 130 // Callback to receive result of a test. 131 void GTestCallback( 132 const TestInfo& test_info, 133 int exit_code, 134 const base::TimeDelta& elapsed_time, 135 bool was_timeout, 136 const std::string& output); 137 138 content::TestLauncherDelegate* launcher_delegate_; 139 140 // Number of times a test timeout occurred. 141 size_t timeout_count_; 142 143 // True after a message about too many timeouts has been printed, 144 // to avoid doing it more than once. 145 bool printed_timeout_message_; 146 147 base::ParallelTestLauncher parallel_launcher_; 148 149 // Store all tests to run before running any of them to properly 150 // handle PRE_ tests. The map is indexed by test full name (e.g. "A.B"). 151 typedef std::map<std::string, TestInfo> TestInfoMap; 152 TestInfoMap tests_to_run_; 153 154 // Temporary directory for user data directories. 155 base::ScopedTempDir temp_dir_; 156 157 DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate); 158}; 159 160void WrapperTestLauncherDelegate::OnTestIterationStarting() { 161 tests_to_run_.clear(); 162} 163 164std::string WrapperTestLauncherDelegate::GetTestNameForFiltering( 165 const testing::TestCase* test_case, 166 const testing::TestInfo* test_info) { 167 return RemoveAnyPrePrefixes( 168 std::string(test_case->name()) + "." + test_info->name()); 169} 170 171bool WrapperTestLauncherDelegate::ShouldRunTest( 172 const testing::TestCase* test_case, 173 const testing::TestInfo* test_info) { 174 if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) && 175 !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) { 176 return false; 177 } 178 179 // Stop test execution after too many timeouts. 180 if (timeout_count_ > 5) { 181 if (!printed_timeout_message_) { 182 printed_timeout_message_ = true; 183 printf("Too many timeouts, aborting test\n"); 184 } 185 return false; 186 } 187 188 return true; 189} 190 191void WrapperTestLauncherDelegate::RunTest( 192 const testing::TestCase* test_case, 193 const testing::TestInfo* test_info, 194 const base::TestLauncherDelegate::TestResultCallback& callback) { 195 TestInfo run_test_info; 196 run_test_info.test_case_name = test_case->name(); 197 run_test_info.test_name = test_info->name(); 198 run_test_info.callbacks.push_back(callback); 199 200 DCHECK(!ContainsKey(tests_to_run_, run_test_info.GetFullName())); 201 tests_to_run_[run_test_info.GetFullName()] = run_test_info; 202} 203 204void WrapperTestLauncherDelegate::RunRemainingTests() { 205 // PRE_ tests and tests that depend on them must share the same 206 // data directory. Using test name as directory name leads to too long 207 // names (exceeding UNIX_PATH_MAX, which creates a problem with 208 // process_singleton_linux). Create a randomly-named temporary directory 209 // and keep track of the names so that PRE_ tests can still re-use them. 210 std::map<std::string, base::FilePath> temp_directories; 211 212 // List of tests we can kick off right now, depending on no other tests. 213 std::vector<std::pair<std::string, CommandLine> > tests_to_run_now; 214 215 for (TestInfoMap::iterator i = tests_to_run_.begin(); 216 i != tests_to_run_.end(); 217 ++i) { 218 const TestInfo& test_info = i->second; 219 220 // Make sure PRE_ tests and tests that depend on them share the same 221 // data directory - based it on the test name without prefixes. 222 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_info.GetFullName())); 223 if (!ContainsKey(temp_directories, test_name_no_pre)) { 224 base::FilePath temp_dir; 225 CHECK(file_util::CreateTemporaryDirInDir( 226 temp_dir_.path(), FILE_PATH_LITERAL("d"), &temp_dir)); 227 temp_directories[test_name_no_pre] = temp_dir; 228 } 229 230 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 231 CHECK(launcher_delegate_->AdjustChildProcessCommandLine( 232 &new_cmd_line, temp_directories[test_name_no_pre])); 233 234 std::string pre_test_name( 235 test_info.test_case_name + "." + kPreTestPrefix + test_info.test_name); 236 if (ContainsKey(tests_to_run_, pre_test_name)) { 237 tests_to_run_[pre_test_name].callbacks.push_back( 238 base::Bind(&WrapperTestLauncherDelegate::RunDependentTest, 239 base::Unretained(this), 240 test_info.GetFullName(), 241 new_cmd_line)); 242 } else { 243 tests_to_run_now.push_back( 244 std::make_pair(test_info.GetFullName(), new_cmd_line)); 245 } 246 } 247 248 for (size_t i = 0; i < tests_to_run_now.size(); i++) { 249 const TestInfo& test_info = tests_to_run_[tests_to_run_now[i].first]; 250 const CommandLine& cmd_line = tests_to_run_now[i].second; 251 DoRunTest(test_info, cmd_line); 252 } 253} 254 255void WrapperTestLauncherDelegate::DoRunTest(const TestInfo& test_info, 256 const CommandLine& command_line) { 257 CommandLine new_cmd_line(command_line.GetProgram()); 258 CommandLine::SwitchMap switches = command_line.GetSwitches(); 259 260 // Strip out gtest_output flag because otherwise we would overwrite results 261 // of the other tests. 262 switches.erase(base::kGTestOutputFlag); 263 264 for (CommandLine::SwitchMap::const_iterator iter = switches.begin(); 265 iter != switches.end(); ++iter) { 266 new_cmd_line.AppendSwitchNative(iter->first, iter->second); 267 } 268 269 // Always enable disabled tests. This method is not called with disabled 270 // tests unless this flag was specified to the browser test executable. 271 new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests"); 272 new_cmd_line.AppendSwitchASCII( 273 "gtest_filter", 274 test_info.test_case_name + "." + test_info.test_name); 275 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 276 277 char* browser_wrapper = getenv("BROWSER_WRAPPER"); 278 279 // PRE_ tests and tests that depend on them should share the sequence token 280 // name, so that they are run serially. 281 std::string test_name_no_pre = RemoveAnyPrePrefixes( 282 test_info.test_case_name + "." + test_info.test_name); 283 284 parallel_launcher_.LaunchChildGTestProcess( 285 new_cmd_line, 286 browser_wrapper ? browser_wrapper : std::string(), 287 TestTimeouts::action_max_timeout(), 288 base::Bind(&WrapperTestLauncherDelegate::GTestCallback, 289 base::Unretained(this), 290 test_info)); 291} 292 293void WrapperTestLauncherDelegate::RunDependentTest( 294 const std::string test_name, 295 const CommandLine& command_line, 296 const base::TestResult& pre_test_result) { 297 const TestInfo& test_info = tests_to_run_[test_name]; 298 if (pre_test_result.status == base::TestResult::TEST_SUCCESS) { 299 // Only run the dependent test if PRE_ test succeeded. 300 DoRunTest(test_info, command_line); 301 } else { 302 // Otherwise skip the test. 303 base::TestResult test_result; 304 test_result.test_case_name = test_info.test_case_name; 305 test_result.test_name = test_info.test_name; 306 test_result.status = base::TestResult::TEST_SKIPPED; 307 for (size_t i = 0; i < test_info.callbacks.size(); i++) 308 test_info.callbacks[i].Run(test_result); 309 } 310} 311 312void WrapperTestLauncherDelegate::GTestCallback( 313 const TestInfo& test_info, 314 int exit_code, 315 const base::TimeDelta& elapsed_time, 316 bool was_timeout, 317 const std::string& output) { 318 base::TestResult result; 319 result.test_case_name = test_info.test_case_name; 320 result.test_name = test_info.test_name; 321 322 // TODO(phajdan.jr): Recognize crashes. 323 if (exit_code == 0) 324 result.status = base::TestResult::TEST_SUCCESS; 325 else if (was_timeout) 326 result.status = base::TestResult::TEST_TIMEOUT; 327 else 328 result.status = base::TestResult::TEST_FAILURE; 329 330 result.elapsed_time = elapsed_time; 331 332 // TODO(phajdan.jr): Use base::PrintTestOutputSnippetOnFailure after migrating 333 // away from run_test_cases.py (http://crbug.com/236893). 334 fprintf(stdout, "%s", output.c_str()); 335 fflush(stdout); 336 337 for (size_t i = 0; i < test_info.callbacks.size(); i++) 338 test_info.callbacks[i].Run(result); 339 parallel_launcher_.ResetOutputWatchdog(); 340} 341 342bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 343 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 344 return true; 345 346 std::string switch_value = 347 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 348 if (!base::StringToInt(switch_value, result) || *result < 1) { 349 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 350 return false; 351 } 352 353 return true; 354} 355 356} // namespace 357 358const char kHelpFlag[] = "help"; 359 360const char kLaunchAsBrowser[] = "as-browser"; 361 362// See kManualTestPrefix above. 363const char kRunManualTestsFlag[] = "run-manual"; 364 365const char kSingleProcessTestsFlag[] = "single_process"; 366 367 368TestLauncherDelegate::~TestLauncherDelegate() { 369} 370 371bool ShouldRunContentMain() { 372#if defined(OS_WIN) || defined(OS_LINUX) 373 CommandLine* command_line = CommandLine::ForCurrentProcess(); 374 return command_line->HasSwitch(switches::kProcessType) || 375 command_line->HasSwitch(kLaunchAsBrowser); 376#else 377 return false; 378#endif // defined(OS_WIN) || defined(OS_LINUX) 379} 380 381int RunContentMain(int argc, char** argv, 382 TestLauncherDelegate* launcher_delegate) { 383#if defined(OS_WIN) 384 sandbox::SandboxInterfaceInfo sandbox_info = {0}; 385 InitializeSandboxInfo(&sandbox_info); 386 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 387 launcher_delegate->CreateContentMainDelegate()); 388 return ContentMain(GetModuleHandle(NULL), 389 &sandbox_info, 390 chrome_main_delegate.get()); 391#elif defined(OS_LINUX) 392 scoped_ptr<ContentMainDelegate> chrome_main_delegate( 393 launcher_delegate->CreateContentMainDelegate()); 394 return ContentMain(argc, const_cast<const char**>(argv), 395 chrome_main_delegate.get()); 396#endif // defined(OS_WIN) 397 NOTREACHED(); 398 return 0; 399} 400 401int LaunchTests(TestLauncherDelegate* launcher_delegate, 402 int argc, 403 char** argv) { 404 DCHECK(!g_launcher_delegate); 405 g_launcher_delegate = launcher_delegate; 406 407 CommandLine::Init(argc, argv); 408 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 409 410 if (command_line->HasSwitch(kHelpFlag)) { 411 PrintUsage(); 412 return 0; 413 } 414 415 if (command_line->HasSwitch(kSingleProcessTestsFlag) || 416 (command_line->HasSwitch(switches::kSingleProcess) && 417 command_line->HasSwitch(base::kGTestFilterFlag)) || 418 command_line->HasSwitch(base::kGTestListTestsFlag) || 419 command_line->HasSwitch(base::kGTestHelpFlag)) { 420#if defined(OS_WIN) 421 if (command_line->HasSwitch(kSingleProcessTestsFlag)) { 422 sandbox::SandboxInterfaceInfo sandbox_info; 423 InitializeSandboxInfo(&sandbox_info); 424 InitializeSandbox(&sandbox_info); 425 } 426#endif 427 return launcher_delegate->RunTestSuite(argc, argv); 428 } 429 430 if (ShouldRunContentMain()) 431 return RunContentMain(argc, argv, launcher_delegate); 432 433 fprintf(stdout, 434 "Starting tests...\n" 435 "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n" 436 "For debugging a test inside a debugger, use the\n" 437 "--gtest_filter=<your_test_name> flag along with either\n" 438 "--single_process (to run the test in one launcher/browser process) or\n" 439 "--single-process (to do the above, and also run Chrome in single-" 440 "process mode).\n"); 441 442 base::AtExitManager at_exit; 443 testing::InitGoogleTest(&argc, argv); 444 TestTimeouts::Initialize(); 445 446 int jobs = 1; // TODO(phajdan.jr): Default to half the number of CPU cores. 447 if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs)) 448 return 1; 449 450 base::MessageLoopForIO message_loop; 451 452 WrapperTestLauncherDelegate delegate(launcher_delegate, jobs); 453 return base::LaunchTests(&delegate, argc, argv); 454} 455 456TestLauncherDelegate* GetCurrentTestLauncherDelegate() { 457 return g_launcher_delegate; 458} 459 460} // namespace content 461