13551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Copyright 2013 The Chromium Authors. All rights reserved. 23551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 33551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// found in the LICENSE file. 43551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 54e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/unit_test_launcher.h" 63551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 7424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/bind.h" 8424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/callback_helpers.h" 93551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/command_line.h" 103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/compiler_specific.h" 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/debug/debugger.h" 126e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)#include "base/files/file_util.h" 134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/files/scoped_temp_dir.h" 143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/format_macros.h" 1558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/message_loop/message_loop.h" 163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/stl_util.h" 1758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/strings/string_number_conversions.h" 183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/strings/string_util.h" 1958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/sys_info.h" 203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/test/gtest_xml_util.h" 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/test_launcher.h" 22424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/test/test_switches.h" 233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/test/test_timeouts.h" 245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "base/third_party/dynamic_annotations/dynamic_annotations.h" 2558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/threading/thread_checker.h" 263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "testing/gtest/include/gtest/gtest.h" 273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)namespace base { 293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)namespace { 313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 3258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)// This constant controls how many tests are run in a single batch by default. 3358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)const size_t kDefaultTestBatchLimit = 10; 343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 35d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)const char kHelpFlag[] = "help"; 36d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Flag to run all tests in a single process. 383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)const char kSingleProcessTestsFlag[] = "single-process-tests"; 393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 40d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)void PrintUsage() { 41d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) fprintf(stdout, 42d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "Runs tests using the gtest framework, each batch of tests being\n" 43d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "run in their own process. Supported command-line flags:\n" 44d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "\n" 45a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Common flags:\n" 46d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --gtest_filter=...\n" 47d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Runs a subset of tests (see --gtest_help for more info).\n" 48a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 49d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --help\n" 50d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Shows this message.\n" 51a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 52d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --gtest_help\n" 53a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Shows the gtest help message.\n" 54a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 55a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-jobs=N\n" 56a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Sets the number of parallel test jobs to N.\n" 57a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 58a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --single-process-tests\n" 59a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Runs the tests and the launcher in the same process. Useful\n" 60a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " for debugging a specific test in a debugger.\n" 61a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 62a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Other flags:\n" 63a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-batch-limit=N\n" 64a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Sets the limit of test batch to run in a single process to N.\n" 65a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) " --test-launcher-debug-launcher\n" 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) " Disables autodetection of debuggers and similar tools,\n" 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) " making it possible to use them to debug launcher itself.\n" 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "\n" 70a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-retry-limit=N\n" 71a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Sets the limit of test retries on failures to N.\n" 72a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 73a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-summary-output=PATH\n" 74a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Saves a JSON machine-readable summary of the run.\n" 75a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 76a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-print-test-stdio=auto|always|never\n" 77a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Controls when full test output is printed.\n" 78a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " auto means to print it when the test failed.\n" 79a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 80a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-total-shards=N\n" 81a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Sets the total number of shards to N.\n" 82a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) "\n" 83a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " --test-launcher-shard-index=N\n" 84a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"); 85d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) fflush(stdout); 86d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)} 87d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 883551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Returns command line for child GTest process based on the command line 893551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// of current process. |test_names| is a vector of test full names 903551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// (e.g. "A.B"), |output_file| is path to the GTest XML output file. 913551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)CommandLine GetCommandLineForChildGTestProcess( 923551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const std::vector<std::string>& test_names, 933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const base::FilePath& output_file) { 94424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 953551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 96424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 97424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 983551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 993551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return new_cmd_line; 1013551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 1023551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1033551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)class UnitTestLauncherDelegate : public TestLauncherDelegate { 10458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) public: 105f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) explicit UnitTestLauncherDelegate(size_t batch_limit, bool use_job_objects) 106f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) : batch_limit_(batch_limit), 107f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) use_job_objects_(use_job_objects) { 10858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 10958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 11058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) virtual ~UnitTestLauncherDelegate() { 11158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 11258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 11358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 1143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) private: 1151e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) struct GTestCallbackState { 1161e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestLauncher* test_launcher; 1171e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::vector<std::string> test_names; 1181e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) FilePath output_file; 1193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) }; 1203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) virtual bool ShouldRunTest(const testing::TestCase* test_case, 1223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const testing::TestInfo* test_info) OVERRIDE { 12358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 12458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 1253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // There is no additional logic to disable specific tests. 1263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return true; 1273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 1283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1291e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) virtual size_t RunTests(TestLauncher* test_launcher, 1301e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names) OVERRIDE { 13158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 13258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 1331e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::vector<std::string> batch; 1341e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (size_t i = 0; i < test_names.size(); i++) { 1351e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) batch.push_back(test_names[i]); 1361e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1371e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if (batch.size() >= batch_limit_) { 1381e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) RunBatch(test_launcher, batch); 1391e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) batch.clear(); 1401e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 1411e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 1421e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1431e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) RunBatch(test_launcher, batch); 1443551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1451e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return test_names.size(); 1463551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 1473551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1481e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) virtual size_t RetryTests( 1491e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestLauncher* test_launcher, 1501e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names) OVERRIDE { 1511e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) MessageLoop::current()->PostTask( 1521e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) FROM_HERE, 1531e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Bind(&UnitTestLauncherDelegate::RunSerially, 1541e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Unretained(this), 1551e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_launcher, 1561e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_names)); 1571e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return test_names.size(); 1581e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 159424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 1601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) void RunSerially(TestLauncher* test_launcher, 1611e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names) { 1621e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if (test_names.empty()) 1633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return; 1643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::vector<std::string> new_test_names(test_names); 1661e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::string test_name(new_test_names.back()); 1671e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) new_test_names.pop_back(); 1681e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Create a dedicated temporary directory to store the xml result data 1703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // per run to ensure clean state and make it possible to launch multiple 1713551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // processes in parallel. 1723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::FilePath output_file; 173a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 17458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output_file = output_file.AppendASCII("test_results.xml"); 1753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::vector<std::string> current_test_names; 1771e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) current_test_names.push_back(test_name); 1781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) CommandLine cmd_line( 1791e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) GetCommandLineForChildGTestProcess(current_test_names, output_file)); 1801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1811e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) GTestCallbackState callback_state; 1821e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_launcher = test_launcher; 1831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_names = current_test_names; 1841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.output_file = output_file; 1851e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1860f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) test_launcher->LaunchChildGTestProcess( 1871e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) cmd_line, 1881e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) std::string(), 1891e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestTimeouts::test_launcher_timeout(), 1906e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0, 1911e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Bind(&UnitTestLauncherDelegate::SerialGTestCallback, 1921e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Unretained(this), 1931e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state, 1941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) new_test_names)); 1951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 1961e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 1971e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) void RunBatch(TestLauncher* test_launcher, 1981e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names) { 1991e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 2001e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2011e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if (test_names.empty()) 2021e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return; 2031e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2041e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // Create a dedicated temporary directory to store the xml result data 2051e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // per run to ensure clean state and make it possible to launch multiple 2061e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // processes in parallel. 2071e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) base::FilePath output_file; 208a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 2091e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) output_file = output_file.AppendASCII("test_results.xml"); 2103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 2113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) CommandLine cmd_line( 2123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) GetCommandLineForChildGTestProcess(test_names, output_file)); 2133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 2143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Adjust the timeout depending on how many tests we're running 2153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // (note that e.g. the last batch of tests will be smaller). 2163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Consider an adaptive timeout, which can change 2173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // depending on how many tests ran and how many remain. 2183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Note: do NOT parse child's stdout to do that, it's known to be 2193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // unreliable (e.g. buffering issues can mix up the output). 2203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::TimeDelta timeout = 22168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_names.size() * TestTimeouts::test_launcher_timeout(); 2223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 2231e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) GTestCallbackState callback_state; 2241e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_launcher = test_launcher; 2251e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_names = test_names; 2261e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.output_file = output_file; 2271e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2280f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) test_launcher->LaunchChildGTestProcess( 22958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) cmd_line, 23058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::string(), 23158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) timeout, 2326e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0, 23358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) Bind(&UnitTestLauncherDelegate::GTestCallback, 2341e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Unretained(this), 2351e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state)); 23658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 23758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 2381e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) void GTestCallback(const GTestCallbackState& callback_state, 23958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) int exit_code, 24068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) const TimeDelta& elapsed_time, 24158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool was_timeout, 24258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const std::string& output) { 24358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 244f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::vector<std::string> tests_to_relaunch; 2450f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) ProcessTestResults(callback_state.test_launcher, 2460f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) callback_state.test_names, 2470f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) callback_state.output_file, 2480f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) output, 2490f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) exit_code, 2500f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) was_timeout, 251f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) &tests_to_relaunch); 252f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 253f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Relaunch requested tests in parallel, but only use single 254f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // test per batch for more precise results (crashes, test passes 255f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // but non-zero exit codes etc). 256f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for (size_t i = 0; i < tests_to_relaunch.size(); i++) { 257f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::vector<std::string> batch; 258f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) batch.push_back(tests_to_relaunch[i]); 259f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) RunBatch(callback_state.test_launcher, batch); 260f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 26158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 26258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // The temporary file's directory is also temporary. 2631e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DeleteFile(callback_state.output_file.DirName(), true); 2641e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) } 2651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2661e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) void SerialGTestCallback(const GTestCallbackState& callback_state, 2671e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names, 2681e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) int exit_code, 2691e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const TimeDelta& elapsed_time, 2701e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) bool was_timeout, 2711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::string& output) { 2721e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 273f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::vector<std::string> tests_to_relaunch; 2741e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) bool called_any_callbacks = 2751e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ProcessTestResults(callback_state.test_launcher, 2761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_names, 2771e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.output_file, 2781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) output, 2791e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) exit_code, 2801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) was_timeout, 281f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) &tests_to_relaunch); 2821e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2831e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // There is only one test, there cannot be other tests to relaunch 2841e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // due to a crash. 285f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(tests_to_relaunch.empty()); 2861e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2870f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) // There is only one test, we should have called back with its result. 2880f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) DCHECK(called_any_callbacks); 2891e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2901e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) // The temporary file's directory is also temporary. 2911e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) DeleteFile(callback_state.output_file.DirName(), true); 2921e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2931e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) MessageLoop::current()->PostTask( 2941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) FROM_HERE, 2951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Bind(&UnitTestLauncherDelegate::RunSerially, 2961e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) Unretained(this), 2971e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) callback_state.test_launcher, 2981e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_names)); 29958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 30058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 30158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) static bool ProcessTestResults( 3021e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestLauncher* test_launcher, 3031e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) const std::vector<std::string>& test_names, 304424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) const base::FilePath& output_file, 30558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const std::string& output, 306424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) int exit_code, 30758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool was_timeout, 308f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::vector<std::string>* tests_to_relaunch) { 3093551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::vector<TestResult> test_results; 310424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) bool crashed = false; 311424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) bool have_test_results = 312424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) ProcessGTestOutput(output_file, &test_results, &crashed); 3133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 31458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool called_any_callback = false; 31558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 3163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if (have_test_results) { 3173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Check for duplicates and mismatches between 3183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // the results we got from XML file and tests we intended to run. 319424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) std::map<std::string, TestResult> results_map; 3203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) for (size_t i = 0; i < test_results.size(); i++) 3211e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) results_map[test_results[i].full_name] = test_results[i]; 322424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 32358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool had_interrupted_test = false; 32458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 325f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Results to be reported back to the test launcher. 326f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::vector<TestResult> final_results; 327f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 3281e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (size_t i = 0; i < test_names.size(); i++) { 3291e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if (ContainsKey(results_map, test_names[i])) { 3301e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) TestResult test_result = results_map[test_names[i]]; 33158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (test_result.status == TestResult::TEST_CRASH) { 33258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) had_interrupted_test = true; 33358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 33458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (was_timeout) { 33558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // Fix up the test status: we forcibly kill the child process 33658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // after the timeout, so from XML results it looks just like 33758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // a crash. 33858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_TIMEOUT; 33958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 34068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) } else if (test_result.status == TestResult::TEST_SUCCESS || 34168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_result.status == TestResult::TEST_FAILURE) { 34268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // We run multiple tests in a batch with a timeout applied 34368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // to the entire batch. It is possible that with other tests 34468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // running quickly some tests take longer than the per-test timeout. 34568043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // For consistent handling of tests independent of order and other 34668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // factors, mark them as timing out. 34768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) if (test_result.elapsed_time > 34868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) TestTimeouts::test_launcher_timeout()) { 34968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_result.status = TestResult::TEST_TIMEOUT; 35068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) } 35158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 3524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result.output_snippet = 3534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) GetTestOutputSnippet(test_result, output); 354f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) final_results.push_back(test_result); 35558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } else if (had_interrupted_test) { 356f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests_to_relaunch->push_back(test_names[i]); 357424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) } else { 358424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // TODO(phajdan.jr): Explicitly pass the info that the test didn't 359424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // run for a mysterious reason. 3601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) LOG(ERROR) << "no test result for " << test_names[i]; 361424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) TestResult test_result; 3621e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_result.full_name = test_names[i]; 36358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_UNKNOWN; 3644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result.output_snippet = 3654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) GetTestOutputSnippet(test_result, output); 366f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) final_results.push_back(test_result); 367424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) } 3683551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 3693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 370424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // TODO(phajdan.jr): Handle the case where processing XML output 371424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // indicates a crash but none of the test results is marked as crashing. 372424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 373f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (final_results.empty()) 374f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return false; 375f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 376f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) bool has_non_success_test = false; 377f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for (size_t i = 0; i < final_results.size(); i++) { 378f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (final_results[i].status != TestResult::TEST_SUCCESS) { 379f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) has_non_success_test = true; 380f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) break; 381f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 382f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 383f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 384f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (!has_non_success_test && exit_code != 0) { 385f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // This is a bit surprising case: all tests are marked as successful, 386f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // but the exit code was not zero. This can happen e.g. under memory 387f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // tools that report leaks this way. 388f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 389f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (final_results.size() == 1) { 390f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Easy case. One test only so we know the non-zero exit code 391f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // was caused by that one test. 392f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT; 393f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } else { 394f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Harder case. Discard the results and request relaunching all 395f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // tests without batching. This will trigger above branch on 396f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // relaunch leading to more precise results. 397f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) LOG(WARNING) << "Not sure which test caused non-zero exit code, " 398f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) << "relaunching all of them without batching."; 399f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 400f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for (size_t i = 0; i < final_results.size(); i++) 401f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) tests_to_relaunch->push_back(final_results[i].full_name); 402f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 403f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return false; 404f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 405f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 406f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 407f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for (size_t i = 0; i < final_results.size(); i++) { 408f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Fix the output snippet after possible changes to the test result. 409f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) final_results[i].output_snippet = 410f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) GetTestOutputSnippet(final_results[i], output); 411f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) test_launcher->OnTestFinished(final_results[i]); 412f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) called_any_callback = true; 413f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 4143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } else { 41558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) fprintf(stdout, 41658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "Failed to get out-of-band test success data, " 41758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "dumping full stdio below:\n%s\n", 41858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output.c_str()); 41958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) fflush(stdout); 42058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 4213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // We do not have reliable details about test results (parsing test 4223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // stdout is known to be unreliable), apply the executable exit code 4233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // to all tests. 4243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 4253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // individually. 4261e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for (size_t i = 0; i < test_names.size(); i++) { 4273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) TestResult test_result; 4281e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_result.full_name = test_names[i]; 42958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_UNKNOWN; 4301e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) test_launcher->OnTestFinished(test_result); 43158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) called_any_callback = true; 4323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 4333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 43458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 43558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return called_any_callback; 4363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 4373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 43868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) ThreadChecker thread_checker_; 43968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 44058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // Maximum number of tests to run in a single batch. 44158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) size_t batch_limit_; 442f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 443f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // Determines whether we use job objects on Windows. 444f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bool use_job_objects_; 4453551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)}; 4463551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 44758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 44858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 44958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return true; 45058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 45158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::string switch_value = 45258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 45358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!StringToInt(switch_value, result) || *result < 1) { 45458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 45558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return false; 45658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 45758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 45858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return true; 45958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)} 46058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 461f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite, 462f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) int default_jobs, 463f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bool use_job_objects, 464f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) const Closure& gtest_init) { 465a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)#if defined(OS_ANDROID) 466a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) // We can't easily fork on Android, just run the test suite directly. 467a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) return run_test_suite.Run(); 468a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)#else 4695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bool force_single_process = false; 4705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch( 4715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) switches::kTestLauncherDebugLauncher)) { 4725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fprintf(stdout, "Forcing test launcher debugging mode.\n"); 4735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fflush(stdout); 4745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } else { 4755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (base::debug::BeingDebugged()) { 4765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fprintf(stdout, 4775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "Debugger detected, switching to single process mode.\n" 4785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "Pass --test-launcher-debug-launcher to debug the launcher " 4795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "itself.\n"); 4805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fflush(stdout); 4815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) force_single_process = true; 4825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 4835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (RunningOnValgrind()) { 4855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fprintf(stdout, 4865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "Valgrind detected, switching to single process mode.\n" 4875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "Pass --test-launcher-debug-launcher to valgrind the launcher " 4885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) "itself.\n"); 4895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fflush(stdout); 4905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) force_single_process = true; 4915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 4925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 4935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 494d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 4955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) || 4965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 4975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) force_single_process) { 4983551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return run_test_suite.Run(); 4993551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 500a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)#endif 5013551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 502d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 503d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) PrintUsage(); 504d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) return 0; 505d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) } 506d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 5073551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::TimeTicks start_time(base::TimeTicks::Now()); 5083551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 509f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) gtest_init.Run(); 51058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) TestTimeouts::Initialize(); 51158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 51258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) int batch_limit = kDefaultTestBatchLimit; 51358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 51458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return 1; 51558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 5163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fprintf(stdout, 51758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 51858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "own process. For debugging a test inside a debugger, use the\n" 51958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "--gtest_filter=<your_test_name> flag along with\n" 520f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) "--single-process-tests.\n"); 5213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fflush(stdout); 5223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 52358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) MessageLoopForIO message_loop; 5243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 525f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) UnitTestLauncherDelegate delegate(batch_limit, use_job_objects); 526a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) base::TestLauncher launcher(&delegate, default_jobs); 527f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bool success = launcher.Run(); 5283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fprintf(stdout, 5303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) "Tests took %" PRId64 " seconds.\n", 5313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) (base::TimeTicks::Now() - start_time).InSeconds()); 5323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fflush(stdout); 5333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5348bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return (success ? 0 : 1); 5353551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 5363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 537f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void InitGoogleTestChar(int* argc, char** argv) { 538f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) testing::InitGoogleTest(argc, argv); 539f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 540f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 541f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#if defined(OS_WIN) 542f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void InitGoogleTestWChar(int* argc, wchar_t** argv) { 543f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) testing::InitGoogleTest(argc, argv); 544f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 545f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#endif // defined(OS_WIN) 546f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 547a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)} // namespace 548a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 549a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)int LaunchUnitTests(int argc, 550a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) char** argv, 551a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) const RunTestSuiteCallback& run_test_suite) { 552f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) CommandLine::Init(argc, argv); 553a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) return LaunchUnitTestsInternal( 554f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) run_test_suite, 555f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) SysInfo::NumberOfProcessors(), 556f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) true, 557f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) Bind(&InitGoogleTestChar, &argc, argv)); 558a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)} 559a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 560a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)int LaunchUnitTestsSerially(int argc, 561a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) char** argv, 562a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) const RunTestSuiteCallback& run_test_suite) { 563f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) CommandLine::Init(argc, argv); 564f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return LaunchUnitTestsInternal( 565f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) run_test_suite, 566f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 1, 567f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) true, 568f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) Bind(&InitGoogleTestChar, &argc, argv)); 569f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 570f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 571f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#if defined(OS_WIN) 572f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)int LaunchUnitTests(int argc, 573f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) wchar_t** argv, 574f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bool use_job_objects, 575f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) const RunTestSuiteCallback& run_test_suite) { 576f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // Windows CommandLine::Init ignores argv anyway. 577f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) CommandLine::Init(argc, NULL); 578f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return LaunchUnitTestsInternal( 579f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) run_test_suite, 580f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) SysInfo::NumberOfProcessors(), 581f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) use_job_objects, 582f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) Bind(&InitGoogleTestWChar, &argc, argv)); 583a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)} 584f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#endif // defined(OS_WIN) 585a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 586424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)} // namespace base 587