unit_test_launcher.cc revision 4e180b6a0b4720a9b8e9e959a882386f690f08ff 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/test/launcher/unit_test_launcher.h"
6
7#include "base/bind.h"
8#include "base/callback_helpers.h"
9#include "base/command_line.h"
10#include "base/compiler_specific.h"
11#include "base/file_util.h"
12#include "base/files/scoped_temp_dir.h"
13#include "base/format_macros.h"
14#include "base/message_loop/message_loop.h"
15#include "base/stl_util.h"
16#include "base/strings/string_number_conversions.h"
17#include "base/strings/string_util.h"
18#include "base/sys_info.h"
19#include "base/test/gtest_xml_util.h"
20#include "base/test/launcher/parallel_test_launcher.h"
21#include "base/test/launcher/test_launcher.h"
22#include "base/test/test_switches.h"
23#include "base/test/test_timeouts.h"
24#include "base/threading/thread_checker.h"
25#include "testing/gtest/include/gtest/gtest.h"
26
27namespace base {
28
29namespace {
30
31// This constant controls how many tests are run in a single batch by default.
32const size_t kDefaultTestBatchLimit = 10;
33
34const char kHelpFlag[] = "help";
35
36// Flag to enable the new launcher logic.
37// TODO(phajdan.jr): Remove it, http://crbug.com/236893 .
38const char kBraveNewTestLauncherFlag[] = "brave-new-test-launcher";
39
40// Flag to run all tests in a single process.
41const char kSingleProcessTestsFlag[] = "single-process-tests";
42
43void PrintUsage() {
44  fprintf(stdout,
45          "Runs tests using the gtest framework, each batch of tests being\n"
46          "run in their own process. Supported command-line flags:\n"
47          "\n"
48          "  --single-process-tests\n"
49          "    Runs the tests and the launcher in the same process. Useful\n"
50          "    for debugging a specific test in a debugger.\n"
51          "  --test-launcher-jobs=N\n"
52          "    Sets the number of parallel test jobs to N.\n"
53          "  --test-launcher-batch-limit=N\n"
54          "    Sets the limit of test batch to run in a single process to N.\n"
55          "  --gtest_filter=...\n"
56          "    Runs a subset of tests (see --gtest_help for more info).\n"
57          "  --help\n"
58          "    Shows this message.\n"
59          "  --gtest_help\n"
60          "    Shows the gtest help message.\n");
61  fflush(stdout);
62}
63
64// Returns command line for child GTest process based on the command line
65// of current process. |test_names| is a vector of test full names
66// (e.g. "A.B"), |output_file| is path to the GTest XML output file.
67CommandLine GetCommandLineForChildGTestProcess(
68    const std::vector<std::string>& test_names,
69    const base::FilePath& output_file) {
70  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
71
72  new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
73  new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
74  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
75  new_cmd_line.AppendSwitch(kBraveNewTestLauncherFlag);
76
77  return new_cmd_line;
78}
79
80class UnitTestLauncherDelegate : public TestLauncherDelegate {
81 public:
82  UnitTestLauncherDelegate(size_t jobs, size_t batch_limit)
83      : parallel_launcher_(jobs),
84        batch_limit_(batch_limit) {
85  }
86
87  virtual ~UnitTestLauncherDelegate() {
88    DCHECK(thread_checker_.CalledOnValidThread());
89  }
90
91 private:
92  struct TestLaunchInfo {
93    std::string GetFullName() const {
94      return test_case_name + "." + test_name;
95    }
96
97    std::string test_case_name;
98    std::string test_name;
99    TestResultCallback callback;
100  };
101
102  virtual void OnTestIterationStarting() OVERRIDE {
103    // Nothing to do.
104  }
105
106  virtual std::string GetTestNameForFiltering(
107      const testing::TestCase* test_case,
108      const testing::TestInfo* test_info) OVERRIDE {
109    DCHECK(thread_checker_.CalledOnValidThread());
110
111    return std::string(test_case->name()) + "." + test_info->name();
112  }
113
114  virtual bool ShouldRunTest(const testing::TestCase* test_case,
115                             const testing::TestInfo* test_info) OVERRIDE {
116    DCHECK(thread_checker_.CalledOnValidThread());
117
118    // There is no additional logic to disable specific tests.
119    return true;
120  }
121
122  virtual void RunTest(const testing::TestCase* test_case,
123                       const testing::TestInfo* test_info,
124                       const TestResultCallback& callback) OVERRIDE {
125    DCHECK(thread_checker_.CalledOnValidThread());
126
127    TestLaunchInfo launch_info;
128    launch_info.test_case_name = test_case->name();
129    launch_info.test_name = test_info->name();
130    launch_info.callback = callback;
131    tests_.push_back(launch_info);
132
133    // Run tests in batches no larger than the limit.
134    if (tests_.size() >= batch_limit_)
135      RunRemainingTests();
136  }
137
138  virtual void RunRemainingTests() OVERRIDE {
139    DCHECK(thread_checker_.CalledOnValidThread());
140
141    if (tests_.empty())
142      return;
143
144    // Create a dedicated temporary directory to store the xml result data
145    // per run to ensure clean state and make it possible to launch multiple
146    // processes in parallel.
147    base::FilePath output_file;
148    CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(),
149                                            &output_file));
150    output_file = output_file.AppendASCII("test_results.xml");
151
152    std::vector<std::string> test_names;
153    for (size_t i = 0; i < tests_.size(); i++)
154      test_names.push_back(tests_[i].GetFullName());
155
156    CommandLine cmd_line(
157        GetCommandLineForChildGTestProcess(test_names, output_file));
158
159    // Adjust the timeout depending on how many tests we're running
160    // (note that e.g. the last batch of tests will be smaller).
161    // TODO(phajdan.jr): Consider an adaptive timeout, which can change
162    // depending on how many tests ran and how many remain.
163    // Note: do NOT parse child's stdout to do that, it's known to be
164    // unreliable (e.g. buffering issues can mix up the output).
165    base::TimeDelta timeout =
166        test_names.size() * TestTimeouts::test_launcher_timeout();
167
168    parallel_launcher_.LaunchChildGTestProcess(
169        cmd_line,
170        std::string(),
171        timeout,
172        Bind(&UnitTestLauncherDelegate::GTestCallback,
173             base::Unretained(this),
174             tests_,
175             output_file));
176    tests_.clear();
177  }
178
179  void GTestCallback(const std::vector<TestLaunchInfo>& tests,
180                     const FilePath& output_file,
181                     int exit_code,
182                     const TimeDelta& elapsed_time,
183                     bool was_timeout,
184                     const std::string& output) {
185    DCHECK(thread_checker_.CalledOnValidThread());
186    std::vector<TestLaunchInfo> tests_to_relaunch_after_interruption;
187    bool called_any_callbacks =
188        ProcessTestResults(tests,
189                           output_file,
190                           output,
191                           exit_code,
192                           was_timeout,
193                           &tests_to_relaunch_after_interruption);
194
195    for (size_t i = 0; i < tests_to_relaunch_after_interruption.size(); i++)
196      tests_.push_back(tests_to_relaunch_after_interruption[i]);
197    RunRemainingTests();
198
199    if (called_any_callbacks)
200      parallel_launcher_.ResetOutputWatchdog();
201
202    // The temporary file's directory is also temporary.
203    DeleteFile(output_file.DirName(), true);
204  }
205
206  static bool ProcessTestResults(
207      const std::vector<TestLaunchInfo>& tests,
208      const base::FilePath& output_file,
209      const std::string& output,
210      int exit_code,
211      bool was_timeout,
212      std::vector<TestLaunchInfo>* tests_to_relaunch_after_interruption) {
213    std::vector<TestResult> test_results;
214    bool crashed = false;
215    bool have_test_results =
216        ProcessGTestOutput(output_file, &test_results, &crashed);
217
218    bool called_any_callback = false;
219
220    if (have_test_results) {
221      // TODO(phajdan.jr): Check for duplicates and mismatches between
222      // the results we got from XML file and tests we intended to run.
223      std::map<std::string, TestResult> results_map;
224      for (size_t i = 0; i < test_results.size(); i++)
225        results_map[test_results[i].GetFullName()] = test_results[i];
226
227      bool had_interrupted_test = false;
228
229      for (size_t i = 0; i < tests.size(); i++) {
230        if (ContainsKey(results_map, tests[i].GetFullName())) {
231          TestResult test_result = results_map[tests[i].GetFullName()];
232          if (test_result.status == TestResult::TEST_CRASH) {
233            had_interrupted_test = true;
234
235            if (was_timeout) {
236              // Fix up the test status: we forcibly kill the child process
237              // after the timeout, so from XML results it looks just like
238              // a crash.
239              test_result.status = TestResult::TEST_TIMEOUT;
240            }
241          } else if (test_result.status == TestResult::TEST_SUCCESS ||
242                     test_result.status == TestResult::TEST_FAILURE) {
243            // We run multiple tests in a batch with a timeout applied
244            // to the entire batch. It is possible that with other tests
245            // running quickly some tests take longer than the per-test timeout.
246            // For consistent handling of tests independent of order and other
247            // factors, mark them as timing out.
248            if (test_result.elapsed_time >
249                TestTimeouts::test_launcher_timeout()) {
250              test_result.status = TestResult::TEST_TIMEOUT;
251            }
252          }
253          test_result.output_snippet =
254              GetTestOutputSnippet(test_result, output);
255          tests[i].callback.Run(test_result);
256          called_any_callback = true;
257        } else if (had_interrupted_test) {
258          tests_to_relaunch_after_interruption->push_back(tests[i]);
259        } else {
260          // TODO(phajdan.jr): Explicitly pass the info that the test didn't
261          // run for a mysterious reason.
262          LOG(ERROR) << "no test result for " << tests[i].GetFullName();
263          TestResult test_result;
264          test_result.test_case_name = tests[i].test_case_name;
265          test_result.test_name = tests[i].test_name;
266          test_result.status = TestResult::TEST_UNKNOWN;
267          test_result.output_snippet =
268              GetTestOutputSnippet(test_result, output);
269          tests[i].callback.Run(test_result);
270          called_any_callback = true;
271        }
272      }
273
274      // TODO(phajdan.jr): Handle the case where processing XML output
275      // indicates a crash but none of the test results is marked as crashing.
276
277      // TODO(phajdan.jr): Handle the case where the exit code is non-zero
278      // but results file indicates that all tests passed (e.g. crash during
279      // shutdown).
280    } else {
281      fprintf(stdout,
282              "Failed to get out-of-band test success data, "
283              "dumping full stdio below:\n%s\n",
284              output.c_str());
285      fflush(stdout);
286
287      // We do not have reliable details about test results (parsing test
288      // stdout is known to be unreliable), apply the executable exit code
289      // to all tests.
290      // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
291      // individually.
292      for (size_t i = 0; i < tests.size(); i++) {
293        TestResult test_result;
294        test_result.test_case_name = tests[i].test_case_name;
295        test_result.test_name = tests[i].test_name;
296        test_result.status = TestResult::TEST_UNKNOWN;
297        tests[i].callback.Run(test_result);
298        called_any_callback = true;
299      }
300    }
301
302    return called_any_callback;
303  }
304
305  ThreadChecker thread_checker_;
306
307  ParallelTestLauncher parallel_launcher_;
308
309  // Maximum number of tests to run in a single batch.
310  size_t batch_limit_;
311
312  std::vector<TestLaunchInfo> tests_;
313};
314
315bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
316  if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
317    return true;
318
319  std::string switch_value =
320      CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
321  if (!StringToInt(switch_value, result) || *result < 1) {
322    LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
323    return false;
324  }
325
326  return true;
327}
328
329}  // namespace
330
331int LaunchUnitTests(int argc,
332                    char** argv,
333                    const RunTestSuiteCallback& run_test_suite) {
334  CommandLine::Init(argc, argv);
335  if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
336      CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
337      !CommandLine::ForCurrentProcess()->HasSwitch(kBraveNewTestLauncherFlag)) {
338    return run_test_suite.Run();
339  }
340
341  if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
342    PrintUsage();
343    return 0;
344  }
345
346  base::TimeTicks start_time(base::TimeTicks::Now());
347
348  testing::InitGoogleTest(&argc, argv);
349  TestTimeouts::Initialize();
350
351  int jobs = SysInfo::NumberOfProcessors();
352  if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs))
353    return 1;
354
355  int batch_limit = kDefaultTestBatchLimit;
356  if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
357    return 1;
358
359  fprintf(stdout,
360          "Starting tests (using %d parallel jobs)...\n"
361          "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
362          "own process. For debugging a test inside a debugger, use the\n"
363          "--gtest_filter=<your_test_name> flag along with\n"
364          "--single-process-tests.\n", jobs);
365  fflush(stdout);
366
367  MessageLoopForIO message_loop;
368
369  base::UnitTestLauncherDelegate delegate(jobs, batch_limit);
370  int exit_code = base::LaunchTests(&delegate, argc, argv);
371
372  fprintf(stdout,
373          "Tests took %" PRId64 " seconds.\n",
374          (base::TimeTicks::Now() - start_time).InSeconds());
375  fflush(stdout);
376
377  return exit_code;
378}
379
380}  // namespace base
381