unit_test_launcher.cc revision a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/test/launcher/unit_test_launcher.h"
6
7#include "base/bind.h"
8#include "base/callback_helpers.h"
9#include "base/command_line.h"
10#include "base/compiler_specific.h"
11#include "base/file_util.h"
12#include "base/files/scoped_temp_dir.h"
13#include "base/format_macros.h"
14#include "base/message_loop/message_loop.h"
15#include "base/stl_util.h"
16#include "base/strings/string_number_conversions.h"
17#include "base/strings/string_util.h"
18#include "base/sys_info.h"
19#include "base/test/gtest_xml_util.h"
20#include "base/test/launcher/test_launcher.h"
21#include "base/test/test_switches.h"
22#include "base/test/test_timeouts.h"
23#include "base/threading/thread_checker.h"
24#include "testing/gtest/include/gtest/gtest.h"
25
26namespace base {
27
28namespace {
29
30// This constant controls how many tests are run in a single batch by default.
31const size_t kDefaultTestBatchLimit = 10;
32
33const char kHelpFlag[] = "help";
34
35// Flag to run all tests in a single process.
36const char kSingleProcessTestsFlag[] = "single-process-tests";
37
38void PrintUsage() {
39  fprintf(stdout,
40          "Runs tests using the gtest framework, each batch of tests being\n"
41          "run in their own process. Supported command-line flags:\n"
42          "\n"
43          " Common flags:\n"
44          "  --gtest_filter=...\n"
45          "    Runs a subset of tests (see --gtest_help for more info).\n"
46          "\n"
47          "  --help\n"
48          "    Shows this message.\n"
49          "\n"
50          "  --gtest_help\n"
51          "    Shows the gtest help message.\n"
52          "\n"
53          "  --test-launcher-jobs=N\n"
54          "    Sets the number of parallel test jobs to N.\n"
55          "\n"
56          "  --single-process-tests\n"
57          "    Runs the tests and the launcher in the same process. Useful\n"
58          "    for debugging a specific test in a debugger.\n"
59          "\n"
60          " Other flags:\n"
61          "  --test-launcher-batch-limit=N\n"
62          "    Sets the limit of test batch to run in a single process to N.\n"
63          "\n"
64          "  --test-launcher-retry-limit=N\n"
65          "    Sets the limit of test retries on failures to N.\n"
66          "\n"
67          "  --test-launcher-summary-output=PATH\n"
68          "    Saves a JSON machine-readable summary of the run.\n"
69          "\n"
70          "  --test-launcher-print-test-stdio=auto|always|never\n"
71          "    Controls when full test output is printed.\n"
72          "    auto means to print it when the test failed.\n"
73          "\n"
74          "  --test-launcher-total-shards=N\n"
75          "    Sets the total number of shards to N.\n"
76          "\n"
77          "  --test-launcher-shard-index=N\n"
78          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
79  fflush(stdout);
80}
81
82// Returns command line for child GTest process based on the command line
83// of current process. |test_names| is a vector of test full names
84// (e.g. "A.B"), |output_file| is path to the GTest XML output file.
85CommandLine GetCommandLineForChildGTestProcess(
86    const std::vector<std::string>& test_names,
87    const base::FilePath& output_file) {
88  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
89
90  new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
91  new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
92  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
93
94  return new_cmd_line;
95}
96
97class UnitTestLauncherDelegate : public TestLauncherDelegate {
98 public:
99  explicit UnitTestLauncherDelegate(size_t batch_limit)
100      : batch_limit_(batch_limit) {
101  }
102
103  virtual ~UnitTestLauncherDelegate() {
104    DCHECK(thread_checker_.CalledOnValidThread());
105  }
106
107 private:
108  struct GTestCallbackState {
109    TestLauncher* test_launcher;
110    std::vector<std::string> test_names;
111    FilePath output_file;
112  };
113
114  virtual void OnTestIterationStarting() OVERRIDE {
115    // Nothing to do.
116  }
117
118  virtual std::string GetTestNameForFiltering(
119      const testing::TestCase* test_case,
120      const testing::TestInfo* test_info) OVERRIDE {
121    DCHECK(thread_checker_.CalledOnValidThread());
122
123    return std::string(test_case->name()) + "." + test_info->name();
124  }
125
126  virtual bool ShouldRunTest(const testing::TestCase* test_case,
127                             const testing::TestInfo* test_info) OVERRIDE {
128    DCHECK(thread_checker_.CalledOnValidThread());
129
130    // There is no additional logic to disable specific tests.
131    return true;
132  }
133
134  virtual size_t RunTests(TestLauncher* test_launcher,
135                          const std::vector<std::string>& test_names) OVERRIDE {
136    DCHECK(thread_checker_.CalledOnValidThread());
137
138    std::vector<std::string> batch;
139    for (size_t i = 0; i < test_names.size(); i++) {
140      batch.push_back(test_names[i]);
141
142      if (batch.size() >= batch_limit_) {
143        RunBatch(test_launcher, batch);
144        batch.clear();
145      }
146    }
147
148    RunBatch(test_launcher, batch);
149
150    return test_names.size();
151  }
152
153  virtual size_t RetryTests(
154      TestLauncher* test_launcher,
155      const std::vector<std::string>& test_names) OVERRIDE {
156    MessageLoop::current()->PostTask(
157        FROM_HERE,
158        Bind(&UnitTestLauncherDelegate::RunSerially,
159             Unretained(this),
160             test_launcher,
161             test_names));
162    return test_names.size();
163  }
164
165  void RunSerially(TestLauncher* test_launcher,
166                   const std::vector<std::string>& test_names) {
167    if (test_names.empty())
168      return;
169
170    std::vector<std::string> new_test_names(test_names);
171    std::string test_name(new_test_names.back());
172    new_test_names.pop_back();
173
174    // Create a dedicated temporary directory to store the xml result data
175    // per run to ensure clean state and make it possible to launch multiple
176    // processes in parallel.
177    base::FilePath output_file;
178    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
179    output_file = output_file.AppendASCII("test_results.xml");
180
181    std::vector<std::string> current_test_names;
182    current_test_names.push_back(test_name);
183    CommandLine cmd_line(
184        GetCommandLineForChildGTestProcess(current_test_names, output_file));
185
186    GTestCallbackState callback_state;
187    callback_state.test_launcher = test_launcher;
188    callback_state.test_names = current_test_names;
189    callback_state.output_file = output_file;
190
191    test_launcher->LaunchChildGTestProcess(
192        cmd_line,
193        std::string(),
194        TestTimeouts::test_launcher_timeout(),
195        Bind(&UnitTestLauncherDelegate::SerialGTestCallback,
196             Unretained(this),
197             callback_state,
198             new_test_names));
199  }
200
201  void RunBatch(TestLauncher* test_launcher,
202                const std::vector<std::string>& test_names) {
203    DCHECK(thread_checker_.CalledOnValidThread());
204
205    if (test_names.empty())
206      return;
207
208    // Create a dedicated temporary directory to store the xml result data
209    // per run to ensure clean state and make it possible to launch multiple
210    // processes in parallel.
211    base::FilePath output_file;
212    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
213    output_file = output_file.AppendASCII("test_results.xml");
214
215    CommandLine cmd_line(
216        GetCommandLineForChildGTestProcess(test_names, output_file));
217
218    // Adjust the timeout depending on how many tests we're running
219    // (note that e.g. the last batch of tests will be smaller).
220    // TODO(phajdan.jr): Consider an adaptive timeout, which can change
221    // depending on how many tests ran and how many remain.
222    // Note: do NOT parse child's stdout to do that, it's known to be
223    // unreliable (e.g. buffering issues can mix up the output).
224    base::TimeDelta timeout =
225        test_names.size() * TestTimeouts::test_launcher_timeout();
226
227    GTestCallbackState callback_state;
228    callback_state.test_launcher = test_launcher;
229    callback_state.test_names = test_names;
230    callback_state.output_file = output_file;
231
232    test_launcher->LaunchChildGTestProcess(
233        cmd_line,
234        std::string(),
235        timeout,
236        Bind(&UnitTestLauncherDelegate::GTestCallback,
237             Unretained(this),
238             callback_state));
239  }
240
241  void GTestCallback(const GTestCallbackState& callback_state,
242                     int exit_code,
243                     const TimeDelta& elapsed_time,
244                     bool was_timeout,
245                     const std::string& output) {
246    DCHECK(thread_checker_.CalledOnValidThread());
247    std::vector<std::string> tests_to_relaunch;
248    ProcessTestResults(callback_state.test_launcher,
249                       callback_state.test_names,
250                       callback_state.output_file,
251                       output,
252                       exit_code,
253                       was_timeout,
254                       &tests_to_relaunch);
255
256    // Relaunch requested tests in parallel, but only use single
257    // test per batch for more precise results (crashes, test passes
258    // but non-zero exit codes etc).
259    for (size_t i = 0; i < tests_to_relaunch.size(); i++) {
260      std::vector<std::string> batch;
261      batch.push_back(tests_to_relaunch[i]);
262      RunBatch(callback_state.test_launcher, batch);
263    }
264
265    // The temporary file's directory is also temporary.
266    DeleteFile(callback_state.output_file.DirName(), true);
267  }
268
269  void SerialGTestCallback(const GTestCallbackState& callback_state,
270                           const std::vector<std::string>& test_names,
271                           int exit_code,
272                           const TimeDelta& elapsed_time,
273                           bool was_timeout,
274                           const std::string& output) {
275    DCHECK(thread_checker_.CalledOnValidThread());
276    std::vector<std::string> tests_to_relaunch;
277    bool called_any_callbacks =
278        ProcessTestResults(callback_state.test_launcher,
279                           callback_state.test_names,
280                           callback_state.output_file,
281                           output,
282                           exit_code,
283                           was_timeout,
284                           &tests_to_relaunch);
285
286    // There is only one test, there cannot be other tests to relaunch
287    // due to a crash.
288    DCHECK(tests_to_relaunch.empty());
289
290    // There is only one test, we should have called back with its result.
291    DCHECK(called_any_callbacks);
292
293    // The temporary file's directory is also temporary.
294    DeleteFile(callback_state.output_file.DirName(), true);
295
296    MessageLoop::current()->PostTask(
297        FROM_HERE,
298        Bind(&UnitTestLauncherDelegate::RunSerially,
299             Unretained(this),
300             callback_state.test_launcher,
301             test_names));
302  }
303
304  static bool ProcessTestResults(
305      TestLauncher* test_launcher,
306      const std::vector<std::string>& test_names,
307      const base::FilePath& output_file,
308      const std::string& output,
309      int exit_code,
310      bool was_timeout,
311      std::vector<std::string>* tests_to_relaunch) {
312    std::vector<TestResult> test_results;
313    bool crashed = false;
314    bool have_test_results =
315        ProcessGTestOutput(output_file, &test_results, &crashed);
316
317    bool called_any_callback = false;
318
319    if (have_test_results) {
320      // TODO(phajdan.jr): Check for duplicates and mismatches between
321      // the results we got from XML file and tests we intended to run.
322      std::map<std::string, TestResult> results_map;
323      for (size_t i = 0; i < test_results.size(); i++)
324        results_map[test_results[i].full_name] = test_results[i];
325
326      bool had_interrupted_test = false;
327
328      // Results to be reported back to the test launcher.
329      std::vector<TestResult> final_results;
330
331      for (size_t i = 0; i < test_names.size(); i++) {
332        if (ContainsKey(results_map, test_names[i])) {
333          TestResult test_result = results_map[test_names[i]];
334          if (test_result.status == TestResult::TEST_CRASH) {
335            had_interrupted_test = true;
336
337            if (was_timeout) {
338              // Fix up the test status: we forcibly kill the child process
339              // after the timeout, so from XML results it looks just like
340              // a crash.
341              test_result.status = TestResult::TEST_TIMEOUT;
342            }
343          } else if (test_result.status == TestResult::TEST_SUCCESS ||
344                     test_result.status == TestResult::TEST_FAILURE) {
345            // We run multiple tests in a batch with a timeout applied
346            // to the entire batch. It is possible that with other tests
347            // running quickly some tests take longer than the per-test timeout.
348            // For consistent handling of tests independent of order and other
349            // factors, mark them as timing out.
350            if (test_result.elapsed_time >
351                TestTimeouts::test_launcher_timeout()) {
352              test_result.status = TestResult::TEST_TIMEOUT;
353            }
354          }
355          test_result.output_snippet =
356              GetTestOutputSnippet(test_result, output);
357          final_results.push_back(test_result);
358        } else if (had_interrupted_test) {
359          tests_to_relaunch->push_back(test_names[i]);
360        } else {
361          // TODO(phajdan.jr): Explicitly pass the info that the test didn't
362          // run for a mysterious reason.
363          LOG(ERROR) << "no test result for " << test_names[i];
364          TestResult test_result;
365          test_result.full_name = test_names[i];
366          test_result.status = TestResult::TEST_UNKNOWN;
367          test_result.output_snippet =
368              GetTestOutputSnippet(test_result, output);
369          final_results.push_back(test_result);
370        }
371      }
372
373      // TODO(phajdan.jr): Handle the case where processing XML output
374      // indicates a crash but none of the test results is marked as crashing.
375
376      if (final_results.empty())
377        return false;
378
379      bool has_non_success_test = false;
380      for (size_t i = 0; i < final_results.size(); i++) {
381        if (final_results[i].status != TestResult::TEST_SUCCESS) {
382          has_non_success_test = true;
383          break;
384        }
385      }
386
387      if (!has_non_success_test && exit_code != 0) {
388        // This is a bit surprising case: all tests are marked as successful,
389        // but the exit code was not zero. This can happen e.g. under memory
390        // tools that report leaks this way.
391
392        if (final_results.size() == 1) {
393          // Easy case. One test only so we know the non-zero exit code
394          // was caused by that one test.
395          final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT;
396        } else {
397          // Harder case. Discard the results and request relaunching all
398          // tests without batching. This will trigger above branch on
399          // relaunch leading to more precise results.
400          LOG(WARNING) << "Not sure which test caused non-zero exit code, "
401                       << "relaunching all of them without batching.";
402
403          for (size_t i = 0; i < final_results.size(); i++)
404            tests_to_relaunch->push_back(final_results[i].full_name);
405
406          return false;
407        }
408      }
409
410      for (size_t i = 0; i < final_results.size(); i++) {
411        // Fix the output snippet after possible changes to the test result.
412        final_results[i].output_snippet =
413            GetTestOutputSnippet(final_results[i], output);
414        test_launcher->OnTestFinished(final_results[i]);
415        called_any_callback = true;
416      }
417    } else {
418      fprintf(stdout,
419              "Failed to get out-of-band test success data, "
420              "dumping full stdio below:\n%s\n",
421              output.c_str());
422      fflush(stdout);
423
424      // We do not have reliable details about test results (parsing test
425      // stdout is known to be unreliable), apply the executable exit code
426      // to all tests.
427      // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
428      // individually.
429      for (size_t i = 0; i < test_names.size(); i++) {
430        TestResult test_result;
431        test_result.full_name = test_names[i];
432        test_result.status = TestResult::TEST_UNKNOWN;
433        test_launcher->OnTestFinished(test_result);
434        called_any_callback = true;
435      }
436    }
437
438    return called_any_callback;
439  }
440
441  ThreadChecker thread_checker_;
442
443  // Maximum number of tests to run in a single batch.
444  size_t batch_limit_;
445};
446
447bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
448  if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
449    return true;
450
451  std::string switch_value =
452      CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
453  if (!StringToInt(switch_value, result) || *result < 1) {
454    LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
455    return false;
456  }
457
458  return true;
459}
460
461int LaunchUnitTestsInternal(int argc,
462                            char** argv,
463                            const RunTestSuiteCallback& run_test_suite,
464                            int default_jobs) {
465  CommandLine::Init(argc, argv);
466#if defined(OS_ANDROID)
467  // We can't easily fork on Android, just run the test suite directly.
468  return run_test_suite.Run();
469#else
470  if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
471      CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag)) {
472    return run_test_suite.Run();
473  }
474#endif
475
476  if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
477    PrintUsage();
478    return 0;
479  }
480
481  base::TimeTicks start_time(base::TimeTicks::Now());
482
483  testing::InitGoogleTest(&argc, argv);
484  TestTimeouts::Initialize();
485
486  int batch_limit = kDefaultTestBatchLimit;
487  if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
488    return 1;
489
490  fprintf(stdout,
491          "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
492          "own process. For debugging a test inside a debugger, use the\n"
493          "--gtest_filter=<your_test_name> flag along with\n"
494          "--single-process-tests.\n");
495  fflush(stdout);
496
497  MessageLoopForIO message_loop;
498
499  UnitTestLauncherDelegate delegate(batch_limit);
500  base::TestLauncher launcher(&delegate, default_jobs);
501  bool success = launcher.Run(argc, argv);
502
503  fprintf(stdout,
504          "Tests took %" PRId64 " seconds.\n",
505          (base::TimeTicks::Now() - start_time).InSeconds());
506  fflush(stdout);
507
508  return (success ? 0 : 1);
509}
510
511}  // namespace
512
513int LaunchUnitTests(int argc,
514                    char** argv,
515                    const RunTestSuiteCallback& run_test_suite) {
516  return LaunchUnitTestsInternal(
517      argc, argv, run_test_suite, SysInfo::NumberOfProcessors());
518}
519
520int LaunchUnitTestsSerially(int argc,
521                            char** argv,
522                            const RunTestSuiteCallback& run_test_suite) {
523  return LaunchUnitTestsInternal(argc, argv, run_test_suite, 1);
524}
525
526}  // namespace base
527