1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 6#define BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 7 8#include <set> 9#include <string> 10 11#include "base/basictypes.h" 12#include "base/callback_forward.h" 13#include "base/compiler_specific.h" 14#include "base/test/launcher/test_result.h" 15#include "base/test/launcher/test_results_tracker.h" 16#include "base/time/time.h" 17#include "base/timer/timer.h" 18 19class CommandLine; 20 21namespace testing { 22class TestCase; 23class TestInfo; 24} 25 26namespace base { 27 28struct LaunchOptions; 29class SequencedWorkerPoolOwner; 30class TestLauncher; 31 32// Constants for GTest command-line flags. 33extern const char kGTestFilterFlag[]; 34extern const char kGTestHelpFlag[]; 35extern const char kGTestListTestsFlag[]; 36extern const char kGTestRepeatFlag[]; 37extern const char kGTestRunDisabledTestsFlag[]; 38extern const char kGTestOutputFlag[]; 39 40// Interface for use with LaunchTests that abstracts away exact details 41// which tests and how are run. 42class TestLauncherDelegate { 43 public: 44 // Called at the start of each test iteration. 45 virtual void OnTestIterationStarting() = 0; 46 47 // Called to get a test name for filtering purposes. Usually it's 48 // test case's name and test's name joined by a dot (e.g. 49 // "TestCaseName.TestName"). 50 // TODO(phajdan.jr): Remove after transitioning away from run_test_cases.py, 51 // http://crbug.com/236893 . 52 virtual std::string GetTestNameForFiltering( 53 const testing::TestCase* test_case, 54 const testing::TestInfo* test_info) = 0; 55 56 // Called before a test is considered for running. If it returns false, 57 // the test is not run. If it returns true, the test will be run provided 58 // it is part of the current shard. 59 virtual bool ShouldRunTest(const testing::TestCase* test_case, 60 const testing::TestInfo* test_info) = 0; 61 62 // Called to make the delegate run the specified tests. The delegate must 63 // return the number of actual tests it's going to run (can be smaller, 64 // equal to, or larger than size of |test_names|). It must also call 65 // |test_launcher|'s OnTestFinished method once per every run test, 66 // regardless of its success. 67 virtual size_t RunTests(TestLauncher* test_launcher, 68 const std::vector<std::string>& test_names) = 0; 69 70 // Called to make the delegate retry the specified tests. The delegate must 71 // return the number of actual tests it's going to retry (can be smaller, 72 // equal to, or larger than size of |test_names|). It must also call 73 // |test_launcher|'s OnTestFinished method once per every retried test, 74 // regardless of its success. 75 virtual size_t RetryTests(TestLauncher* test_launcher, 76 const std::vector<std::string>& test_names) = 0; 77 78 protected: 79 virtual ~TestLauncherDelegate(); 80}; 81 82// Launches tests using a TestLauncherDelegate. 83class TestLauncher { 84 public: 85 // Constructor. |parallel_jobs| is the limit of simultaneous parallel test 86 // jobs. 87 TestLauncher(TestLauncherDelegate* launcher_delegate, size_t parallel_jobs); 88 ~TestLauncher(); 89 90 // Runs the launcher. Must be called at most once. 91 bool Run(int argc, char** argv) WARN_UNUSED_RESULT; 92 93 // Callback called after a child process finishes. First argument is the exit 94 // code, second one is child process elapsed time, third one is true if 95 // the child process was terminated because of a timeout, and fourth one 96 // contains output of the child (stdout and stderr together). 97 typedef Callback<void(int, const TimeDelta&, bool, const std::string&)> 98 LaunchChildGTestProcessCallback; 99 100 // Launches a child process (assumed to be gtest-based binary) using 101 // |command_line|. If |wrapper| is not empty, it is prepended to the final 102 // command line. If the child process is still running after |timeout|, it 103 // is terminated. After the child process finishes |callback| is called 104 // on the same thread this method was called. 105 void LaunchChildGTestProcess(const CommandLine& command_line, 106 const std::string& wrapper, 107 base::TimeDelta timeout, 108 const LaunchChildGTestProcessCallback& callback); 109 110 // Called when a test has finished running. 111 void OnTestFinished(const TestResult& result); 112 113 private: 114 bool Init() WARN_UNUSED_RESULT; 115 116 // Runs all tests in current iteration. Uses callbacks to communicate success. 117 void RunTests(); 118 119 void RunTestIteration(); 120 121 // Saves test results summary as JSON if requested from command line. 122 void MaybeSaveSummaryAsJSON(); 123 124 // Called on a worker thread after a child process finishes. 125 void OnLaunchTestProcessFinished( 126 const LaunchChildGTestProcessCallback& callback, 127 int exit_code, 128 const TimeDelta& elapsed_time, 129 bool was_timeout, 130 const std::string& output); 131 132 // Called by the delay timer when no output was made for a while. 133 void OnOutputTimeout(); 134 135 // Make sure we don't accidentally call the wrong methods e.g. on the worker 136 // pool thread. With lots of callbacks used this is non-trivial. 137 // Should be the first member so that it's destroyed last: when destroying 138 // other members, especially the worker pool, we may check the code is running 139 // on the correct thread. 140 ThreadChecker thread_checker_; 141 142 TestLauncherDelegate* launcher_delegate_; 143 144 // Support for outer sharding, just like gtest does. 145 int32 total_shards_; // Total number of outer shards, at least one. 146 int32 shard_index_; // Index of shard the launcher is to run. 147 148 int cycles_; // Number of remaining test itreations, or -1 for infinite. 149 150 // Test filters (empty means no filter). Entries are separated by colons. 151 std::string positive_test_filter_; 152 std::string negative_test_filter_; 153 154 // Number of tests started in this iteration. 155 size_t test_started_count_; 156 157 // Number of tests finished in this iteration. 158 size_t test_finished_count_; 159 160 // Number of tests successfully finished in this iteration. 161 size_t test_success_count_; 162 163 // Number of tests either timing out or having an unknown result, 164 // likely indicating a more systemic problem if widespread. 165 size_t test_broken_count_; 166 167 // Number of retries in this iteration. 168 size_t retry_count_; 169 170 // Maximum number of retries per iteration. 171 size_t retry_limit_; 172 173 // Tests to retry in this iteration. 174 std::set<std::string> tests_to_retry_; 175 176 // Result to be returned from Run. 177 bool run_result_; 178 179 TestResultsTracker results_tracker_; 180 181 // Watchdog timer to make sure we do not go without output for too long. 182 DelayTimer<TestLauncher> watchdog_timer_; 183 184 // Number of jobs to run in parallel. 185 size_t parallel_jobs_; 186 187 // Worker pool used to launch processes in parallel. 188 scoped_ptr<SequencedWorkerPoolOwner> worker_pool_owner_; 189 190 DISALLOW_COPY_AND_ASSIGN(TestLauncher); 191}; 192 193// Extract part from |full_output| that applies to |result|. 194std::string GetTestOutputSnippet(const TestResult& result, 195 const std::string& full_output); 196 197// Launches a child process (assumed to be gtest-based binary) 198// using |command_line|. If |wrapper| is not empty, it is prepended 199// to the final command line. If the child process is still running 200// after |timeout|, it is terminated and |*was_timeout| is set to true. 201int LaunchChildGTestProcess(const CommandLine& command_line, 202 const std::string& wrapper, 203 base::TimeDelta timeout, 204 bool* was_timeout); 205 206// Returns command line command line after gtest-specific processing 207// and applying |wrapper|. 208CommandLine PrepareCommandLineForGTest(const CommandLine& command_line, 209 const std::string& wrapper); 210 211// Launches a child process using |command_line|. If the child process is still 212// running after |timeout|, it is terminated and |*was_timeout| is set to true. 213// Returns exit code of the process. 214int LaunchChildTestProcessWithOptions(const CommandLine& command_line, 215 const LaunchOptions& options, 216 base::TimeDelta timeout, 217 bool* was_timeout); 218 219} // namespace base 220 221#endif // BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 222