1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
6#define BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
7
8#include <set>
9#include <string>
10
11#include "base/basictypes.h"
12#include "base/callback_forward.h"
13#include "base/compiler_specific.h"
14#include "base/test/launcher/test_result.h"
15#include "base/test/launcher/test_results_tracker.h"
16#include "base/time/time.h"
17#include "base/timer/timer.h"
18
19class CommandLine;
20
21namespace testing {
22class TestCase;
23class TestInfo;
24}
25
26namespace base {
27
28struct LaunchOptions;
29class SequencedWorkerPoolOwner;
30class TestLauncher;
31
32// Constants for GTest command-line flags.
33extern const char kGTestFilterFlag[];
34extern const char kGTestHelpFlag[];
35extern const char kGTestListTestsFlag[];
36extern const char kGTestRepeatFlag[];
37extern const char kGTestRunDisabledTestsFlag[];
38extern const char kGTestOutputFlag[];
39
40// Interface for use with LaunchTests that abstracts away exact details
41// which tests and how are run.
42class TestLauncherDelegate {
43 public:
44  // Called at the start of each test iteration.
45  virtual void OnTestIterationStarting() = 0;
46
47  // Called to get a test name for filtering purposes. Usually it's
48  // test case's name and test's name joined by a dot (e.g.
49  // "TestCaseName.TestName").
50  // TODO(phajdan.jr): Remove after transitioning away from run_test_cases.py,
51  // http://crbug.com/236893 .
52  virtual std::string GetTestNameForFiltering(
53      const testing::TestCase* test_case,
54      const testing::TestInfo* test_info) = 0;
55
56  // Called before a test is considered for running. If it returns false,
57  // the test is not run. If it returns true, the test will be run provided
58  // it is part of the current shard.
59  virtual bool ShouldRunTest(const testing::TestCase* test_case,
60                             const testing::TestInfo* test_info) = 0;
61
62  // Called to make the delegate run the specified tests. The delegate must
63  // return the number of actual tests it's going to run (can be smaller,
64  // equal to, or larger than size of |test_names|). It must also call
65  // |test_launcher|'s OnTestFinished method once per every run test,
66  // regardless of its success.
67  virtual size_t RunTests(TestLauncher* test_launcher,
68                          const std::vector<std::string>& test_names) = 0;
69
70  // Called to make the delegate retry the specified tests. The delegate must
71  // return the number of actual tests it's going to retry (can be smaller,
72  // equal to, or larger than size of |test_names|). It must also call
73  // |test_launcher|'s OnTestFinished method once per every retried test,
74  // regardless of its success.
75  virtual size_t RetryTests(TestLauncher* test_launcher,
76                            const std::vector<std::string>& test_names) = 0;
77
78 protected:
79  virtual ~TestLauncherDelegate();
80};
81
82// Launches tests using a TestLauncherDelegate.
83class TestLauncher {
84 public:
85  // Constructor. |parallel_jobs| is the limit of simultaneous parallel test
86  // jobs.
87  TestLauncher(TestLauncherDelegate* launcher_delegate, size_t parallel_jobs);
88  ~TestLauncher();
89
90  // Runs the launcher. Must be called at most once.
91  bool Run(int argc, char** argv) WARN_UNUSED_RESULT;
92
93  // Callback called after a child process finishes. First argument is the exit
94  // code, second one is child process elapsed time, third one is true if
95  // the child process was terminated because of a timeout, and fourth one
96  // contains output of the child (stdout and stderr together).
97  typedef Callback<void(int, const TimeDelta&, bool, const std::string&)>
98      LaunchChildGTestProcessCallback;
99
100  // Launches a child process (assumed to be gtest-based binary) using
101  // |command_line|. If |wrapper| is not empty, it is prepended to the final
102  // command line. If the child process is still running after |timeout|, it
103  // is terminated. After the child process finishes |callback| is called
104  // on the same thread this method was called.
105  void LaunchChildGTestProcess(const CommandLine& command_line,
106                               const std::string& wrapper,
107                               base::TimeDelta timeout,
108                               const LaunchChildGTestProcessCallback& callback);
109
110  // Called when a test has finished running.
111  void OnTestFinished(const TestResult& result);
112
113 private:
114  bool Init() WARN_UNUSED_RESULT;
115
116  // Runs all tests in current iteration. Uses callbacks to communicate success.
117  void RunTests();
118
119  void RunTestIteration();
120
121  // Saves test results summary as JSON if requested from command line.
122  void MaybeSaveSummaryAsJSON();
123
124  // Called on a worker thread after a child process finishes.
125  void OnLaunchTestProcessFinished(
126      const LaunchChildGTestProcessCallback& callback,
127      int exit_code,
128      const TimeDelta& elapsed_time,
129      bool was_timeout,
130      const std::string& output);
131
132  // Called by the delay timer when no output was made for a while.
133  void OnOutputTimeout();
134
135  // Make sure we don't accidentally call the wrong methods e.g. on the worker
136  // pool thread. With lots of callbacks used this is non-trivial.
137  // Should be the first member so that it's destroyed last: when destroying
138  // other members, especially the worker pool, we may check the code is running
139  // on the correct thread.
140  ThreadChecker thread_checker_;
141
142  TestLauncherDelegate* launcher_delegate_;
143
144  // Support for outer sharding, just like gtest does.
145  int32 total_shards_;  // Total number of outer shards, at least one.
146  int32 shard_index_;   // Index of shard the launcher is to run.
147
148  int cycles_;  // Number of remaining test itreations, or -1 for infinite.
149
150  // Test filters (empty means no filter). Entries are separated by colons.
151  std::string positive_test_filter_;
152  std::string negative_test_filter_;
153
154  // Number of tests started in this iteration.
155  size_t test_started_count_;
156
157  // Number of tests finished in this iteration.
158  size_t test_finished_count_;
159
160  // Number of tests successfully finished in this iteration.
161  size_t test_success_count_;
162
163  // Number of tests either timing out or having an unknown result,
164  // likely indicating a more systemic problem if widespread.
165  size_t test_broken_count_;
166
167  // Number of retries in this iteration.
168  size_t retry_count_;
169
170  // Maximum number of retries per iteration.
171  size_t retry_limit_;
172
173  // Tests to retry in this iteration.
174  std::set<std::string> tests_to_retry_;
175
176  // Result to be returned from Run.
177  bool run_result_;
178
179  TestResultsTracker results_tracker_;
180
181  // Watchdog timer to make sure we do not go without output for too long.
182  DelayTimer<TestLauncher> watchdog_timer_;
183
184  // Number of jobs to run in parallel.
185  size_t parallel_jobs_;
186
187  // Worker pool used to launch processes in parallel.
188  scoped_ptr<SequencedWorkerPoolOwner> worker_pool_owner_;
189
190  DISALLOW_COPY_AND_ASSIGN(TestLauncher);
191};
192
193// Extract part from |full_output| that applies to |result|.
194std::string GetTestOutputSnippet(const TestResult& result,
195                                 const std::string& full_output);
196
197// Launches a child process (assumed to be gtest-based binary)
198// using |command_line|. If |wrapper| is not empty, it is prepended
199// to the final command line. If the child process is still running
200// after |timeout|, it is terminated and |*was_timeout| is set to true.
201int LaunchChildGTestProcess(const CommandLine& command_line,
202                            const std::string& wrapper,
203                            base::TimeDelta timeout,
204                            bool* was_timeout);
205
206// Returns command line command line after gtest-specific processing
207// and applying |wrapper|.
208CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
209                                       const std::string& wrapper);
210
211// Launches a child process using |command_line|. If the child process is still
212// running after |timeout|, it is terminated and |*was_timeout| is set to true.
213// Returns exit code of the process.
214int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
215                                      const LaunchOptions& options,
216                                      base::TimeDelta timeout,
217                                      bool* was_timeout);
218
219}  // namespace base
220
221#endif  // BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
222