1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
6#define BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
7
8#include <set>
9#include <string>
10
11#include "base/basictypes.h"
12#include "base/callback_forward.h"
13#include "base/compiler_specific.h"
14#include "base/test/launcher/test_result.h"
15#include "base/test/launcher/test_results_tracker.h"
16#include "base/time/time.h"
17#include "base/timer/timer.h"
18
19namespace testing {
20class TestCase;
21class TestInfo;
22}
23
24namespace base {
25
26class CommandLine;
27struct LaunchOptions;
28class SequencedWorkerPoolOwner;
29class TestLauncher;
30
31// Constants for GTest command-line flags.
32extern const char kGTestFilterFlag[];
33extern const char kGTestHelpFlag[];
34extern const char kGTestListTestsFlag[];
35extern const char kGTestRepeatFlag[];
36extern const char kGTestRunDisabledTestsFlag[];
37extern const char kGTestOutputFlag[];
38
39// Interface for use with LaunchTests that abstracts away exact details
40// which tests and how are run.
41class TestLauncherDelegate {
42 public:
43  // Called before a test is considered for running. If it returns false,
44  // the test is not run. If it returns true, the test will be run provided
45  // it is part of the current shard.
46  virtual bool ShouldRunTest(const testing::TestCase* test_case,
47                             const testing::TestInfo* test_info) = 0;
48
49  // Called to make the delegate run the specified tests. The delegate must
50  // return the number of actual tests it's going to run (can be smaller,
51  // equal to, or larger than size of |test_names|). It must also call
52  // |test_launcher|'s OnTestFinished method once per every run test,
53  // regardless of its success.
54  virtual size_t RunTests(TestLauncher* test_launcher,
55                          const std::vector<std::string>& test_names) = 0;
56
57  // Called to make the delegate retry the specified tests. The delegate must
58  // return the number of actual tests it's going to retry (can be smaller,
59  // equal to, or larger than size of |test_names|). It must also call
60  // |test_launcher|'s OnTestFinished method once per every retried test,
61  // regardless of its success.
62  virtual size_t RetryTests(TestLauncher* test_launcher,
63                            const std::vector<std::string>& test_names) = 0;
64
65 protected:
66  virtual ~TestLauncherDelegate();
67};
68
69// Launches tests using a TestLauncherDelegate.
70class TestLauncher {
71 public:
72  // Flags controlling behavior of LaunchChildGTestProcess.
73  enum LaunchChildGTestProcessFlags {
74    // Allows usage of job objects on Windows. Helps properly clean up child
75    // processes.
76    USE_JOB_OBJECTS = (1 << 0),
77
78    // Allows breakaway from job on Windows. May result in some child processes
79    // not being properly terminated after launcher dies if these processes
80    // fail to cooperate.
81    ALLOW_BREAKAWAY_FROM_JOB = (1 << 1),
82  };
83
84  // Constructor. |parallel_jobs| is the limit of simultaneous parallel test
85  // jobs.
86  TestLauncher(TestLauncherDelegate* launcher_delegate, size_t parallel_jobs);
87  ~TestLauncher();
88
89  // Runs the launcher. Must be called at most once.
90  bool Run() WARN_UNUSED_RESULT;
91
92  // Callback called after a child process finishes. First argument is the exit
93  // code, second one is child process elapsed time, third one is true if
94  // the child process was terminated because of a timeout, and fourth one
95  // contains output of the child (stdout and stderr together).
96  typedef Callback<void(int, const TimeDelta&, bool, const std::string&)>
97      LaunchChildGTestProcessCallback;
98
99  // Launches a child process (assumed to be gtest-based binary) using
100  // |command_line|. If |wrapper| is not empty, it is prepended to the final
101  // command line. If the child process is still running after |timeout|, it
102  // is terminated. After the child process finishes |callback| is called
103  // on the same thread this method was called.
104  void LaunchChildGTestProcess(const CommandLine& command_line,
105                               const std::string& wrapper,
106                               base::TimeDelta timeout,
107                               int flags,
108                               const LaunchChildGTestProcessCallback& callback);
109
110  // Called when a test has finished running.
111  void OnTestFinished(const TestResult& result);
112
113  // Constructs a full test name given a test case name and a test name.
114  static std::string FormatFullTestName(const std::string& test_case_name,
115                                        const std::string& test_name);
116
117 private:
118  bool Init() WARN_UNUSED_RESULT;
119
120  // Runs all tests in current iteration. Uses callbacks to communicate success.
121  void RunTests();
122
123  void RunTestIteration();
124
125  // Saves test results summary as JSON if requested from command line.
126  void MaybeSaveSummaryAsJSON();
127
128  // Called on a worker thread after a child process finishes.
129  void OnLaunchTestProcessFinished(
130      const LaunchChildGTestProcessCallback& callback,
131      int exit_code,
132      const TimeDelta& elapsed_time,
133      bool was_timeout,
134      const std::string& output);
135
136  // Called when a test iteration is finished.
137  void OnTestIterationFinished();
138
139  // Called by the delay timer when no output was made for a while.
140  void OnOutputTimeout();
141
142  // Make sure we don't accidentally call the wrong methods e.g. on the worker
143  // pool thread. With lots of callbacks used this is non-trivial.
144  // Should be the first member so that it's destroyed last: when destroying
145  // other members, especially the worker pool, we may check the code is running
146  // on the correct thread.
147  ThreadChecker thread_checker_;
148
149  TestLauncherDelegate* launcher_delegate_;
150
151  // Support for outer sharding, just like gtest does.
152  int32 total_shards_;  // Total number of outer shards, at least one.
153  int32 shard_index_;   // Index of shard the launcher is to run.
154
155  int cycles_;  // Number of remaining test itreations, or -1 for infinite.
156
157  // Test filters (empty means no filter).
158  std::vector<std::string> positive_test_filter_;
159  std::vector<std::string> negative_test_filter_;
160
161  // Number of tests started in this iteration.
162  size_t test_started_count_;
163
164  // Number of tests finished in this iteration.
165  size_t test_finished_count_;
166
167  // Number of tests successfully finished in this iteration.
168  size_t test_success_count_;
169
170  // Number of tests either timing out or having an unknown result,
171  // likely indicating a more systemic problem if widespread.
172  size_t test_broken_count_;
173
174  // Number of retries in this iteration.
175  size_t retry_count_;
176
177  // Maximum number of retries per iteration.
178  size_t retry_limit_;
179
180  // Tests to retry in this iteration.
181  std::set<std::string> tests_to_retry_;
182
183  // Result to be returned from Run.
184  bool run_result_;
185
186  TestResultsTracker results_tracker_;
187
188  // Watchdog timer to make sure we do not go without output for too long.
189  DelayTimer<TestLauncher> watchdog_timer_;
190
191  // Number of jobs to run in parallel.
192  size_t parallel_jobs_;
193
194  // Worker pool used to launch processes in parallel.
195  scoped_ptr<SequencedWorkerPoolOwner> worker_pool_owner_;
196
197  DISALLOW_COPY_AND_ASSIGN(TestLauncher);
198};
199
200// Extract part from |full_output| that applies to |result|.
201std::string GetTestOutputSnippet(const TestResult& result,
202                                 const std::string& full_output);
203
204}  // namespace base
205
206#endif  // BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_
207