unit_test_launcher.cc revision f8ee788a64d60abd8f2d742a5fdedde054ecd910
1a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch// Copyright 2013 The Chromium Authors. All rights reserved. 2a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch// Use of this source code is governed by a BSD-style license that can be 3a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch// found in the LICENSE file. 4a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 5a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/test/launcher/unit_test_launcher.h" 6a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 7a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/bind.h" 8a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/callback_helpers.h" 9a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/command_line.h" 10a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/compiler_specific.h" 11a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/debug/debugger.h" 12a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/file_util.h" 13a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/files/scoped_temp_dir.h" 14a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/format_macros.h" 15a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/message_loop/message_loop.h" 16a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/stl_util.h" 17a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/strings/string_number_conversions.h" 18a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/strings/string_util.h" 19a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/sys_info.h" 20a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/test/gtest_xml_util.h" 21a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/test/launcher/test_launcher.h" 22a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/test/test_switches.h" 23a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/test/test_timeouts.h" 24a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/third_party/dynamic_annotations/dynamic_annotations.h" 25a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "base/threading/thread_checker.h" 26a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#include "testing/gtest/include/gtest/gtest.h" 27a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 28a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochnamespace base { 29a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 30a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochnamespace { 31a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 32a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch// This constant controls how many tests are run in a single batch by default. 33a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochconst size_t kDefaultTestBatchLimit = 10; 34a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 35a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochconst char kHelpFlag[] = "help"; 36a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 375d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)// Flag to run all tests in a single process. 38a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochconst char kSingleProcessTestsFlag[] = "single-process-tests"; 39a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 40a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochvoid PrintUsage() { 41a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 42a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Runs tests using the gtest framework, each batch of tests being\n" 43a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "run in their own process. Supported command-line flags:\n" 44c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) "\n" 45a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch " Common flags:\n" 46a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch " --gtest_filter=...\n" 47a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch " Runs a subset of tests (see --gtest_help for more info).\n" 48f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 49f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --help\n" 50f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Shows this message.\n" 51f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 52f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --gtest_help\n" 53f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Shows the gtest help message.\n" 54f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 55f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --test-launcher-jobs=N\n" 56f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Sets the number of parallel test jobs to N.\n" 57f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 58f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --single-process-tests\n" 59f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Runs the tests and the launcher in the same process. Useful\n" 60f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " for debugging a specific test in a debugger.\n" 61f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 62f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Other flags:\n" 63f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --test-launcher-batch-limit=N\n" 64f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Sets the limit of test batch to run in a single process to N.\n" 65f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 66f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --test-launcher-debug-launcher\n" 67f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Disables autodetection of debuggers and similar tools,\n" 68f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " making it possible to use them to debug launcher itself.\n" 69f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu "\n" 70f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " --test-launcher-retry-limit=N\n" 71f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu " Sets the limit of test retries on failures to N.\n" 725d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) "\n" 735d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) " --test-launcher-summary-output=PATH\n" 74197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch " Saves a JSON machine-readable summary of the run.\n" 75197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch "\n" 76197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch " --test-launcher-print-test-stdio=auto|always|never\n" 77197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch " Controls when full test output is printed.\n" 78197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch " auto means to print it when the test failed.\n" 795d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) "\n" 805d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) " --test-launcher-total-shards=N\n" 815d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) " Sets the total number of shards to N.\n" 825d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) "\n" 835d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) " --test-launcher-shard-index=N\n" 845d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"); 855d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) fflush(stdout); 865d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)} 875d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 885d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)// Returns command line for child GTest process based on the command line 895d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)// of current process. |test_names| is a vector of test full names 90197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch// (e.g. "A.B"), |output_file| is path to the GTest XML output file. 91197021e6b966cfb06891637935ef33fff06433d1Ben MurdochCommandLine GetCommandLineForChildGTestProcess( 92197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch const std::vector<std::string>& test_names, 93197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch const base::FilePath& output_file) { 94197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 955d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 965d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 975d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 985d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 995d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 1005d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) return new_cmd_line; 1015d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)} 1025d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 1035d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)class UnitTestLauncherDelegate : public TestLauncherDelegate { 1045d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) public: 105197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch explicit UnitTestLauncherDelegate(size_t batch_limit, bool use_job_objects) 106197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch : batch_limit_(batch_limit), 1075d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) use_job_objects_(use_job_objects) { 1085d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) } 1095d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 1105d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) virtual ~UnitTestLauncherDelegate() { 111a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 112a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 113a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 114a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch private: 115a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch struct GTestCallbackState { 116a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestLauncher* test_launcher; 117a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> test_names; 118a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch FilePath output_file; 119a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch }; 120a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 121a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch virtual bool ShouldRunTest(const testing::TestCase* test_case, 122a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const testing::TestInfo* test_info) OVERRIDE { 123a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 124a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 125a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // There is no additional logic to disable specific tests. 126a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return true; 127a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 128a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 129a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch virtual size_t RunTests(TestLauncher* test_launcher, 130a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names) OVERRIDE { 131a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 132a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 133a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> batch; 134a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch for (size_t i = 0; i < test_names.size(); i++) { 135a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch batch.push_back(test_names[i]); 136a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 137a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (batch.size() >= batch_limit_) { 138a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch RunBatch(test_launcher, batch); 139a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch batch.clear(); 140a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 141a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 142a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 143a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch RunBatch(test_launcher, batch); 144a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 145a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return test_names.size(); 146a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 147a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 148a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch virtual size_t RetryTests( 149a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestLauncher* test_launcher, 150a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names) OVERRIDE { 151a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch MessageLoop::current()->PostTask( 152a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch FROM_HERE, 153a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&UnitTestLauncherDelegate::RunSerially, 154a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Unretained(this), 155a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_launcher, 156a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_names)); 157a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return test_names.size(); 158a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 159a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 160a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch void RunSerially(TestLauncher* test_launcher, 161a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names) { 162a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (test_names.empty()) 163a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return; 164a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 165a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> new_test_names(test_names); 166a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::string test_name(new_test_names.back()); 167a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch new_test_names.pop_back(); 168a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 169a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Create a dedicated temporary directory to store the xml result data 170a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // per run to ensure clean state and make it possible to launch multiple 171a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // processes in parallel. 172a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch base::FilePath output_file; 173a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 174a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch output_file = output_file.AppendASCII("test_results.xml"); 175a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 176a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> current_test_names; 177a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch current_test_names.push_back(test_name); 178a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine cmd_line( 179a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch GetCommandLineForChildGTestProcess(current_test_names, output_file)); 180a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 181a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch GTestCallbackState callback_state; 182a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.test_launcher = test_launcher; 183a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.test_names = current_test_names; 184a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.output_file = output_file; 185a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 186a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_launcher->LaunchChildGTestProcess( 187a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch cmd_line, 188a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::string(), 189a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestTimeouts::test_launcher_timeout(), 190a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch use_job_objects_, 191a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&UnitTestLauncherDelegate::SerialGTestCallback, 192a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Unretained(this), 193a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state, 194a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch new_test_names)); 195a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 196a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 197a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch void RunBatch(TestLauncher* test_launcher, 198a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names) { 199a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 200a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 201a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (test_names.empty()) 202a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return; 203a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 204a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Create a dedicated temporary directory to store the xml result data 205a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // per run to ensure clean state and make it possible to launch multiple 206197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch // processes in parallel. 207a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch base::FilePath output_file; 208a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 209a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch output_file = output_file.AppendASCII("test_results.xml"); 210a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 211a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine cmd_line( 212a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch GetCommandLineForChildGTestProcess(test_names, output_file)); 213a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 214a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Adjust the timeout depending on how many tests we're running 215a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // (note that e.g. the last batch of tests will be smaller). 216a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // TODO(phajdan.jr): Consider an adaptive timeout, which can change 217a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // depending on how many tests ran and how many remain. 218a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Note: do NOT parse child's stdout to do that, it's known to be 219197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch // unreliable (e.g. buffering issues can mix up the output). 220a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch base::TimeDelta timeout = 221a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_names.size() * TestTimeouts::test_launcher_timeout(); 222a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 223197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch GTestCallbackState callback_state; 224197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch callback_state.test_launcher = test_launcher; 225197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch callback_state.test_names = test_names; 226a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.output_file = output_file; 227a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 228a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_launcher->LaunchChildGTestProcess( 229a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch cmd_line, 230a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::string(), 231a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch timeout, 232a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch use_job_objects_, 233a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&UnitTestLauncherDelegate::GTestCallback, 234a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Unretained(this), 235a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state)); 236a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 237197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch 238a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch void GTestCallback(const GTestCallbackState& callback_state, 239a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch int exit_code, 240a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const TimeDelta& elapsed_time, 241a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool was_timeout, 242a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::string& output) { 243a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 244a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> tests_to_relaunch; 245a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch ProcessTestResults(callback_state.test_launcher, 246a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.test_names, 247a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.output_file, 248a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch output, 249a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch exit_code, 250a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch was_timeout, 251a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch &tests_to_relaunch); 252a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 253197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch // Relaunch requested tests in parallel, but only use single 254197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch // test per batch for more precise results (crashes, test passes 255197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch // but non-zero exit codes etc). 256197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch for (size_t i = 0; i < tests_to_relaunch.size(); i++) { 257197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch std::vector<std::string> batch; 258197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch batch.push_back(tests_to_relaunch[i]); 259197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch RunBatch(callback_state.test_launcher, batch); 260197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch } 261a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 262a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // The temporary file's directory is also temporary. 263a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DeleteFile(callback_state.output_file.DirName(), true); 264a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 265a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 266a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch void SerialGTestCallback(const GTestCallbackState& callback_state, 267a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names, 268a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch int exit_code, 269a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const TimeDelta& elapsed_time, 270a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool was_timeout, 271a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::string& output) { 272a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(thread_checker_.CalledOnValidThread()); 273a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string> tests_to_relaunch; 274a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool called_any_callbacks = 275a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch ProcessTestResults(callback_state.test_launcher, 276a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.test_names, 277a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.output_file, 278a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch output, 279197021e6b966cfb06891637935ef33fff06433d1Ben Murdoch exit_code, 280a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch was_timeout, 281a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch &tests_to_relaunch); 282a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 283a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // There is only one test, there cannot be other tests to relaunch 284a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // due to a crash. 285a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(tests_to_relaunch.empty()); 286a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 287a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // There is only one test, we should have called back with its result. 288a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DCHECK(called_any_callbacks); 289a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 290a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // The temporary file's directory is also temporary. 291a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch DeleteFile(callback_state.output_file.DirName(), true); 292a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 293a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch MessageLoop::current()->PostTask( 294a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch FROM_HERE, 295a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&UnitTestLauncherDelegate::RunSerially, 296a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Unretained(this), 297a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch callback_state.test_launcher, 298a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_names)); 299a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 300a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 301a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch static bool ProcessTestResults( 302a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestLauncher* test_launcher, 303a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::vector<std::string>& test_names, 304a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const base::FilePath& output_file, 305a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const std::string& output, 306a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch int exit_code, 307a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool was_timeout, 308a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<std::string>* tests_to_relaunch) { 309a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<TestResult> test_results; 310a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool crashed = false; 311a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool have_test_results = 312f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu ProcessGTestOutput(output_file, &test_results, &crashed); 313a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 314f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu bool called_any_callback = false; 315a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 316a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (have_test_results) { 317a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // TODO(phajdan.jr): Check for duplicates and mismatches between 318a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // the results we got from XML file and tests we intended to run. 3195d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) std::map<std::string, TestResult> results_map; 320a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch for (size_t i = 0; i < test_results.size(); i++) 321a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch results_map[test_results[i].full_name] = test_results[i]; 322a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 323a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool had_interrupted_test = false; 324a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 3255d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) // Results to be reported back to the test launcher. 326a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::vector<TestResult> final_results; 327a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 3285d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) for (size_t i = 0; i < test_names.size(); i++) { 329a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (ContainsKey(results_map, test_names[i])) { 330a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestResult test_result = results_map[test_names[i]]; 3315d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) if (test_result.status == TestResult::TEST_CRASH) { 332a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch had_interrupted_test = true; 333a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 3345d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) if (was_timeout) { 335a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Fix up the test status: we forcibly kill the child process 336a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // after the timeout, so from XML results it looks just like 3375d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) // a crash. 338a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.status = TestResult::TEST_TIMEOUT; 339a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 340a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } else if (test_result.status == TestResult::TEST_SUCCESS || 341a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.status == TestResult::TEST_FAILURE) { 342a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // We run multiple tests in a batch with a timeout applied 3435d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) // to the entire batch. It is possible that with other tests 344a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // running quickly some tests take longer than the per-test timeout. 345a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // For consistent handling of tests independent of order and other 346a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // factors, mark them as timing out. 347a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (test_result.elapsed_time > 348a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestTimeouts::test_launcher_timeout()) { 349a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.status = TestResult::TEST_TIMEOUT; 350a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 351a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 352a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.output_snippet = 353a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch GetTestOutputSnippet(test_result, output); 354a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch final_results.push_back(test_result); 355a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } else if (had_interrupted_test) { 356a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch tests_to_relaunch->push_back(test_names[i]); 357a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } else { 358a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // TODO(phajdan.jr): Explicitly pass the info that the test didn't 359a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // run for a mysterious reason. 360a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch LOG(ERROR) << "no test result for " << test_names[i]; 361a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestResult test_result; 362a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.full_name = test_names[i]; 363a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.status = TestResult::TEST_UNKNOWN; 364a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_result.output_snippet = 365a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch GetTestOutputSnippet(test_result, output); 366f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu final_results.push_back(test_result); 367f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu } 368f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu } 369e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles) 3705d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) // TODO(phajdan.jr): Handle the case where processing XML output 371f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu // indicates a crash but none of the test results is marked as crashing. 372f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu 373f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu if (final_results.empty()) 374a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return false; 375a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 376a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool has_non_success_test = false; 377a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch for (size_t i = 0; i < final_results.size(); i++) { 378a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (final_results[i].status != TestResult::TEST_SUCCESS) { 379a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch has_non_success_test = true; 380a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch break; 381a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 382a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 383a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 384a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (!has_non_success_test && exit_code != 0) { 385a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // This is a bit surprising case: all tests are marked as successful, 386a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // but the exit code was not zero. This can happen e.g. under memory 387a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // tools that report leaks this way. 388a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 389a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (final_results.size() == 1) { 390a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Easy case. One test only so we know the non-zero exit code 391a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // was caused by that one test. 392a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT; 393a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } else { 3945d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) // Harder case. Discard the results and request relaunching all 395a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // tests without batching. This will trigger above branch on 396a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // relaunch leading to more precise results. 397a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch LOG(WARNING) << "Not sure which test caused non-zero exit code, " 398a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch << "relaunching all of them without batching."; 399a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 400a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch for (size_t i = 0; i < final_results.size(); i++) 401a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch tests_to_relaunch->push_back(final_results[i].full_name); 4025d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 403a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return false; 404a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 405a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 406a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 407a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch for (size_t i = 0; i < final_results.size(); i++) { 408a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Fix the output snippet after possible changes to the test result. 409a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch final_results[i].output_snippet = 4105d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) GetTestOutputSnippet(final_results[i], output); 411a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch test_launcher->OnTestFinished(final_results[i]); 412a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch called_any_callback = true; 413a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 414a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } else { 415a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 416a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Failed to get out-of-band test success data, " 417a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "dumping full stdio below:\n%s\n", 4185d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) output.c_str()); 419a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fflush(stdout); 420a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 421a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // We do not have reliable details about test results (parsing test 422a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // stdout is known to be unreliable), apply the executable exit code 423a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // to all tests. 424a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 425e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles) // individually. 4265d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) for (size_t i = 0; i < test_names.size(); i++) { 4275d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) TestResult test_result; 4285d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) test_result.full_name = test_names[i]; 4295d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) test_result.status = TestResult::TEST_UNKNOWN; 4305d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) test_launcher->OnTestFinished(test_result); 4315d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) called_any_callback = true; 4325d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) } 4335d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) } 4345d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 4355d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) return called_any_callback; 436a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 437a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 438a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch ThreadChecker thread_checker_; 439a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 440a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Maximum number of tests to run in a single batch. 441a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch size_t batch_limit_; 442a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 443a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Determines whether we use job objects on Windows. 444a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool use_job_objects_; 445a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch}; 446a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 447a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochbool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 448a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 449a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return true; 450a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 451a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch std::string switch_value = 452a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 453a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (!StringToInt(switch_value, result) || *result < 1) { 454a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 455a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return false; 456a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 457a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 458f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu return true; 459f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu} 460f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu 461f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liuint LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite, 462a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch int default_jobs, 463a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool use_job_objects, 464a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const Closure& gtest_init) { 465a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#if defined(OS_ANDROID) 466a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // We can't easily fork on Android, just run the test suite directly. 467a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return run_test_suite.Run(); 4685d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)#else 4695d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) bool force_single_process = false; 470e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch( 4715d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) switches::kTestLauncherDebugLauncher)) { 4725d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) fprintf(stdout, "Forcing test launcher debugging mode.\n"); 4735d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) fflush(stdout); 4745d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) } else { 475a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (base::debug::BeingDebugged()) { 476a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 477a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Debugger detected, switching to single process mode.\n" 478a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Pass --test-launcher-debug-launcher to debug the launcher " 479a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "itself.\n"); 480a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fflush(stdout); 481a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch force_single_process = true; 482a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch } 483a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 484a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (RunningOnValgrind()) { 485a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 486a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Valgrind detected, switching to single process mode.\n" 487a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Pass --test-launcher-debug-launcher to valgrind the launcher " 488a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "itself.\n"); 489a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fflush(stdout); 490a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch force_single_process = true; 4916f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch } 4926f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch } 4936f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch 4946f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 4956f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) || 4966f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 497323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) force_single_process) { 498323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) return run_test_suite.Run(); 499323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) } 500323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles)#endif 501323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) 502323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 503f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu PrintUsage(); 504f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu return 0; 505f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu } 506a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 507a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch base::TimeTicks start_time(base::TimeTicks::Now()); 508a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 509a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch gtest_init.Run(); 510a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch TestTimeouts::Initialize(); 511a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 512a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch int batch_limit = kDefaultTestBatchLimit; 513a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 514a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return 1; 515a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 516a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 517a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 518a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "own process. For debugging a test inside a debugger, use the\n" 519a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "--gtest_filter=<your_test_name> flag along with\n" 520a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "--single-process-tests.\n"); 521a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fflush(stdout); 522a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 523a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch MessageLoopForIO message_loop; 524a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 525a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch UnitTestLauncherDelegate delegate(batch_limit, use_job_objects); 526a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch base::TestLauncher launcher(&delegate, default_jobs); 527a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool success = launcher.Run(); 528a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 529a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fprintf(stdout, 530a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch "Tests took %" PRId64 " seconds.\n", 531a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch (base::TimeTicks::Now() - start_time).InSeconds()); 532a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch fflush(stdout); 533a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 534a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return (success ? 0 : 1); 535a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 536a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 537a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochvoid InitGoogleTestChar(int* argc, char** argv) { 538a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch testing::InitGoogleTest(argc, argv); 539a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 540a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 541a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#if defined(OS_WIN) 542a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochvoid InitGoogleTestWChar(int* argc, wchar_t** argv) { 543a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch testing::InitGoogleTest(argc, argv); 544a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 545a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#endif // defined(OS_WIN) 546a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 547a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} // namespace 548a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 549a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochint LaunchUnitTests(int argc, 550a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch char** argv, 551a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const RunTestSuiteCallback& run_test_suite) { 552a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine::Init(argc, argv); 553a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return LaunchUnitTestsInternal( 554a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch run_test_suite, 555a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch SysInfo::NumberOfProcessors(), 556a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch true, 557a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&InitGoogleTestChar, &argc, argv)); 558a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 559a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 560a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochint LaunchUnitTestsSerially(int argc, 561a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch char** argv, 562a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const RunTestSuiteCallback& run_test_suite) { 563a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine::Init(argc, argv); 564a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return LaunchUnitTestsInternal( 565a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch run_test_suite, 566a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 1, 567a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch true, 568a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&InitGoogleTestChar, &argc, argv)); 569a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 570a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 571a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#if defined(OS_WIN) 572a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdochint LaunchUnitTests(int argc, 573a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch wchar_t** argv, 574a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch bool use_job_objects, 575a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch const RunTestSuiteCallback& run_test_suite) { 576a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch // Windows CommandLine::Init ignores argv anyway. 577a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch CommandLine::Init(argc, NULL); 578a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch return LaunchUnitTestsInternal( 579a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch run_test_suite, 580a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch SysInfo::NumberOfProcessors(), 581a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch use_job_objects, 582a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch Bind(&InitGoogleTestWChar, &argc, argv)); 583a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} 584a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch#endif // defined(OS_WIN) 585a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch 586a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch} // namespace base 587a9984bf9ddc3cf73fdae3f29134a2bab379e7029Ben Murdoch