unit_test_launcher.cc revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/test/launcher/unit_test_launcher.h"
6
7#include "base/bind.h"
8#include "base/callback_helpers.h"
9#include "base/command_line.h"
10#include "base/compiler_specific.h"
11#include "base/debug/debugger.h"
12#include "base/file_util.h"
13#include "base/files/scoped_temp_dir.h"
14#include "base/format_macros.h"
15#include "base/message_loop/message_loop.h"
16#include "base/stl_util.h"
17#include "base/strings/string_number_conversions.h"
18#include "base/strings/string_util.h"
19#include "base/sys_info.h"
20#include "base/test/gtest_xml_util.h"
21#include "base/test/launcher/test_launcher.h"
22#include "base/test/test_switches.h"
23#include "base/test/test_timeouts.h"
24#include "base/third_party/dynamic_annotations/dynamic_annotations.h"
25#include "base/threading/thread_checker.h"
26#include "testing/gtest/include/gtest/gtest.h"
27
28namespace base {
29
30namespace {
31
32// This constant controls how many tests are run in a single batch by default.
33const size_t kDefaultTestBatchLimit = 10;
34
35const char kHelpFlag[] = "help";
36
37// Flag to run all tests in a single process.
38const char kSingleProcessTestsFlag[] = "single-process-tests";
39
40void PrintUsage() {
41  fprintf(stdout,
42          "Runs tests using the gtest framework, each batch of tests being\n"
43          "run in their own process. Supported command-line flags:\n"
44          "\n"
45          " Common flags:\n"
46          "  --gtest_filter=...\n"
47          "    Runs a subset of tests (see --gtest_help for more info).\n"
48          "\n"
49          "  --help\n"
50          "    Shows this message.\n"
51          "\n"
52          "  --gtest_help\n"
53          "    Shows the gtest help message.\n"
54          "\n"
55          "  --test-launcher-jobs=N\n"
56          "    Sets the number of parallel test jobs to N.\n"
57          "\n"
58          "  --single-process-tests\n"
59          "    Runs the tests and the launcher in the same process. Useful\n"
60          "    for debugging a specific test in a debugger.\n"
61          "\n"
62          " Other flags:\n"
63          "  --test-launcher-batch-limit=N\n"
64          "    Sets the limit of test batch to run in a single process to N.\n"
65          "\n"
66          "  --test-launcher-debug-launcher\n"
67          "    Disables autodetection of debuggers and similar tools,\n"
68          "    making it possible to use them to debug launcher itself.\n"
69          "\n"
70          "  --test-launcher-retry-limit=N\n"
71          "    Sets the limit of test retries on failures to N.\n"
72          "\n"
73          "  --test-launcher-summary-output=PATH\n"
74          "    Saves a JSON machine-readable summary of the run.\n"
75          "\n"
76          "  --test-launcher-print-test-stdio=auto|always|never\n"
77          "    Controls when full test output is printed.\n"
78          "    auto means to print it when the test failed.\n"
79          "\n"
80          "  --test-launcher-total-shards=N\n"
81          "    Sets the total number of shards to N.\n"
82          "\n"
83          "  --test-launcher-shard-index=N\n"
84          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
85  fflush(stdout);
86}
87
88// Returns command line for child GTest process based on the command line
89// of current process. |test_names| is a vector of test full names
90// (e.g. "A.B"), |output_file| is path to the GTest XML output file.
91CommandLine GetCommandLineForChildGTestProcess(
92    const std::vector<std::string>& test_names,
93    const base::FilePath& output_file) {
94  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
95
96  new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
97  new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
98  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
99
100  return new_cmd_line;
101}
102
103class UnitTestLauncherDelegate : public TestLauncherDelegate {
104 public:
105  explicit UnitTestLauncherDelegate(size_t batch_limit)
106      : batch_limit_(batch_limit) {
107  }
108
109  virtual ~UnitTestLauncherDelegate() {
110    DCHECK(thread_checker_.CalledOnValidThread());
111  }
112
113 private:
114  struct GTestCallbackState {
115    TestLauncher* test_launcher;
116    std::vector<std::string> test_names;
117    FilePath output_file;
118  };
119
120  virtual bool ShouldRunTest(const testing::TestCase* test_case,
121                             const testing::TestInfo* test_info) OVERRIDE {
122    DCHECK(thread_checker_.CalledOnValidThread());
123
124    // There is no additional logic to disable specific tests.
125    return true;
126  }
127
128  virtual size_t RunTests(TestLauncher* test_launcher,
129                          const std::vector<std::string>& test_names) OVERRIDE {
130    DCHECK(thread_checker_.CalledOnValidThread());
131
132    std::vector<std::string> batch;
133    for (size_t i = 0; i < test_names.size(); i++) {
134      batch.push_back(test_names[i]);
135
136      if (batch.size() >= batch_limit_) {
137        RunBatch(test_launcher, batch);
138        batch.clear();
139      }
140    }
141
142    RunBatch(test_launcher, batch);
143
144    return test_names.size();
145  }
146
147  virtual size_t RetryTests(
148      TestLauncher* test_launcher,
149      const std::vector<std::string>& test_names) OVERRIDE {
150    MessageLoop::current()->PostTask(
151        FROM_HERE,
152        Bind(&UnitTestLauncherDelegate::RunSerially,
153             Unretained(this),
154             test_launcher,
155             test_names));
156    return test_names.size();
157  }
158
159  void RunSerially(TestLauncher* test_launcher,
160                   const std::vector<std::string>& test_names) {
161    if (test_names.empty())
162      return;
163
164    std::vector<std::string> new_test_names(test_names);
165    std::string test_name(new_test_names.back());
166    new_test_names.pop_back();
167
168    // Create a dedicated temporary directory to store the xml result data
169    // per run to ensure clean state and make it possible to launch multiple
170    // processes in parallel.
171    base::FilePath output_file;
172    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
173    output_file = output_file.AppendASCII("test_results.xml");
174
175    std::vector<std::string> current_test_names;
176    current_test_names.push_back(test_name);
177    CommandLine cmd_line(
178        GetCommandLineForChildGTestProcess(current_test_names, output_file));
179
180    GTestCallbackState callback_state;
181    callback_state.test_launcher = test_launcher;
182    callback_state.test_names = current_test_names;
183    callback_state.output_file = output_file;
184
185    test_launcher->LaunchChildGTestProcess(
186        cmd_line,
187        std::string(),
188        TestTimeouts::test_launcher_timeout(),
189        Bind(&UnitTestLauncherDelegate::SerialGTestCallback,
190             Unretained(this),
191             callback_state,
192             new_test_names));
193  }
194
195  void RunBatch(TestLauncher* test_launcher,
196                const std::vector<std::string>& test_names) {
197    DCHECK(thread_checker_.CalledOnValidThread());
198
199    if (test_names.empty())
200      return;
201
202    // Create a dedicated temporary directory to store the xml result data
203    // per run to ensure clean state and make it possible to launch multiple
204    // processes in parallel.
205    base::FilePath output_file;
206    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
207    output_file = output_file.AppendASCII("test_results.xml");
208
209    CommandLine cmd_line(
210        GetCommandLineForChildGTestProcess(test_names, output_file));
211
212    // Adjust the timeout depending on how many tests we're running
213    // (note that e.g. the last batch of tests will be smaller).
214    // TODO(phajdan.jr): Consider an adaptive timeout, which can change
215    // depending on how many tests ran and how many remain.
216    // Note: do NOT parse child's stdout to do that, it's known to be
217    // unreliable (e.g. buffering issues can mix up the output).
218    base::TimeDelta timeout =
219        test_names.size() * TestTimeouts::test_launcher_timeout();
220
221    GTestCallbackState callback_state;
222    callback_state.test_launcher = test_launcher;
223    callback_state.test_names = test_names;
224    callback_state.output_file = output_file;
225
226    test_launcher->LaunchChildGTestProcess(
227        cmd_line,
228        std::string(),
229        timeout,
230        Bind(&UnitTestLauncherDelegate::GTestCallback,
231             Unretained(this),
232             callback_state));
233  }
234
235  void GTestCallback(const GTestCallbackState& callback_state,
236                     int exit_code,
237                     const TimeDelta& elapsed_time,
238                     bool was_timeout,
239                     const std::string& output) {
240    DCHECK(thread_checker_.CalledOnValidThread());
241    std::vector<std::string> tests_to_relaunch;
242    ProcessTestResults(callback_state.test_launcher,
243                       callback_state.test_names,
244                       callback_state.output_file,
245                       output,
246                       exit_code,
247                       was_timeout,
248                       &tests_to_relaunch);
249
250    // Relaunch requested tests in parallel, but only use single
251    // test per batch for more precise results (crashes, test passes
252    // but non-zero exit codes etc).
253    for (size_t i = 0; i < tests_to_relaunch.size(); i++) {
254      std::vector<std::string> batch;
255      batch.push_back(tests_to_relaunch[i]);
256      RunBatch(callback_state.test_launcher, batch);
257    }
258
259    // The temporary file's directory is also temporary.
260    DeleteFile(callback_state.output_file.DirName(), true);
261  }
262
263  void SerialGTestCallback(const GTestCallbackState& callback_state,
264                           const std::vector<std::string>& test_names,
265                           int exit_code,
266                           const TimeDelta& elapsed_time,
267                           bool was_timeout,
268                           const std::string& output) {
269    DCHECK(thread_checker_.CalledOnValidThread());
270    std::vector<std::string> tests_to_relaunch;
271    bool called_any_callbacks =
272        ProcessTestResults(callback_state.test_launcher,
273                           callback_state.test_names,
274                           callback_state.output_file,
275                           output,
276                           exit_code,
277                           was_timeout,
278                           &tests_to_relaunch);
279
280    // There is only one test, there cannot be other tests to relaunch
281    // due to a crash.
282    DCHECK(tests_to_relaunch.empty());
283
284    // There is only one test, we should have called back with its result.
285    DCHECK(called_any_callbacks);
286
287    // The temporary file's directory is also temporary.
288    DeleteFile(callback_state.output_file.DirName(), true);
289
290    MessageLoop::current()->PostTask(
291        FROM_HERE,
292        Bind(&UnitTestLauncherDelegate::RunSerially,
293             Unretained(this),
294             callback_state.test_launcher,
295             test_names));
296  }
297
298  static bool ProcessTestResults(
299      TestLauncher* test_launcher,
300      const std::vector<std::string>& test_names,
301      const base::FilePath& output_file,
302      const std::string& output,
303      int exit_code,
304      bool was_timeout,
305      std::vector<std::string>* tests_to_relaunch) {
306    std::vector<TestResult> test_results;
307    bool crashed = false;
308    bool have_test_results =
309        ProcessGTestOutput(output_file, &test_results, &crashed);
310
311    bool called_any_callback = false;
312
313    if (have_test_results) {
314      // TODO(phajdan.jr): Check for duplicates and mismatches between
315      // the results we got from XML file and tests we intended to run.
316      std::map<std::string, TestResult> results_map;
317      for (size_t i = 0; i < test_results.size(); i++)
318        results_map[test_results[i].full_name] = test_results[i];
319
320      bool had_interrupted_test = false;
321
322      // Results to be reported back to the test launcher.
323      std::vector<TestResult> final_results;
324
325      for (size_t i = 0; i < test_names.size(); i++) {
326        if (ContainsKey(results_map, test_names[i])) {
327          TestResult test_result = results_map[test_names[i]];
328          if (test_result.status == TestResult::TEST_CRASH) {
329            had_interrupted_test = true;
330
331            if (was_timeout) {
332              // Fix up the test status: we forcibly kill the child process
333              // after the timeout, so from XML results it looks just like
334              // a crash.
335              test_result.status = TestResult::TEST_TIMEOUT;
336            }
337          } else if (test_result.status == TestResult::TEST_SUCCESS ||
338                     test_result.status == TestResult::TEST_FAILURE) {
339            // We run multiple tests in a batch with a timeout applied
340            // to the entire batch. It is possible that with other tests
341            // running quickly some tests take longer than the per-test timeout.
342            // For consistent handling of tests independent of order and other
343            // factors, mark them as timing out.
344            if (test_result.elapsed_time >
345                TestTimeouts::test_launcher_timeout()) {
346              test_result.status = TestResult::TEST_TIMEOUT;
347            }
348          }
349          test_result.output_snippet =
350              GetTestOutputSnippet(test_result, output);
351          final_results.push_back(test_result);
352        } else if (had_interrupted_test) {
353          tests_to_relaunch->push_back(test_names[i]);
354        } else {
355          // TODO(phajdan.jr): Explicitly pass the info that the test didn't
356          // run for a mysterious reason.
357          LOG(ERROR) << "no test result for " << test_names[i];
358          TestResult test_result;
359          test_result.full_name = test_names[i];
360          test_result.status = TestResult::TEST_UNKNOWN;
361          test_result.output_snippet =
362              GetTestOutputSnippet(test_result, output);
363          final_results.push_back(test_result);
364        }
365      }
366
367      // TODO(phajdan.jr): Handle the case where processing XML output
368      // indicates a crash but none of the test results is marked as crashing.
369
370      if (final_results.empty())
371        return false;
372
373      bool has_non_success_test = false;
374      for (size_t i = 0; i < final_results.size(); i++) {
375        if (final_results[i].status != TestResult::TEST_SUCCESS) {
376          has_non_success_test = true;
377          break;
378        }
379      }
380
381      if (!has_non_success_test && exit_code != 0) {
382        // This is a bit surprising case: all tests are marked as successful,
383        // but the exit code was not zero. This can happen e.g. under memory
384        // tools that report leaks this way.
385
386        if (final_results.size() == 1) {
387          // Easy case. One test only so we know the non-zero exit code
388          // was caused by that one test.
389          final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT;
390        } else {
391          // Harder case. Discard the results and request relaunching all
392          // tests without batching. This will trigger above branch on
393          // relaunch leading to more precise results.
394          LOG(WARNING) << "Not sure which test caused non-zero exit code, "
395                       << "relaunching all of them without batching.";
396
397          for (size_t i = 0; i < final_results.size(); i++)
398            tests_to_relaunch->push_back(final_results[i].full_name);
399
400          return false;
401        }
402      }
403
404      for (size_t i = 0; i < final_results.size(); i++) {
405        // Fix the output snippet after possible changes to the test result.
406        final_results[i].output_snippet =
407            GetTestOutputSnippet(final_results[i], output);
408        test_launcher->OnTestFinished(final_results[i]);
409        called_any_callback = true;
410      }
411    } else {
412      fprintf(stdout,
413              "Failed to get out-of-band test success data, "
414              "dumping full stdio below:\n%s\n",
415              output.c_str());
416      fflush(stdout);
417
418      // We do not have reliable details about test results (parsing test
419      // stdout is known to be unreliable), apply the executable exit code
420      // to all tests.
421      // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
422      // individually.
423      for (size_t i = 0; i < test_names.size(); i++) {
424        TestResult test_result;
425        test_result.full_name = test_names[i];
426        test_result.status = TestResult::TEST_UNKNOWN;
427        test_launcher->OnTestFinished(test_result);
428        called_any_callback = true;
429      }
430    }
431
432    return called_any_callback;
433  }
434
435  ThreadChecker thread_checker_;
436
437  // Maximum number of tests to run in a single batch.
438  size_t batch_limit_;
439};
440
441bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
442  if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
443    return true;
444
445  std::string switch_value =
446      CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
447  if (!StringToInt(switch_value, result) || *result < 1) {
448    LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
449    return false;
450  }
451
452  return true;
453}
454
455int LaunchUnitTestsInternal(int argc,
456                            char** argv,
457                            const RunTestSuiteCallback& run_test_suite,
458                            int default_jobs) {
459  CommandLine::Init(argc, argv);
460
461#if defined(OS_ANDROID)
462  // We can't easily fork on Android, just run the test suite directly.
463  return run_test_suite.Run();
464#else
465  bool force_single_process = false;
466  if (CommandLine::ForCurrentProcess()->HasSwitch(
467          switches::kTestLauncherDebugLauncher)) {
468    fprintf(stdout, "Forcing test launcher debugging mode.\n");
469    fflush(stdout);
470  } else {
471    if (base::debug::BeingDebugged()) {
472      fprintf(stdout,
473              "Debugger detected, switching to single process mode.\n"
474              "Pass --test-launcher-debug-launcher to debug the launcher "
475              "itself.\n");
476      fflush(stdout);
477      force_single_process = true;
478    }
479
480    if (RunningOnValgrind()) {
481      fprintf(stdout,
482              "Valgrind detected, switching to single process mode.\n"
483              "Pass --test-launcher-debug-launcher to valgrind the launcher "
484              "itself.\n");
485      fflush(stdout);
486      force_single_process = true;
487    }
488  }
489
490  if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
491      CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
492      CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
493      force_single_process) {
494    return run_test_suite.Run();
495  }
496#endif
497
498  if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
499    PrintUsage();
500    return 0;
501  }
502
503  base::TimeTicks start_time(base::TimeTicks::Now());
504
505  testing::InitGoogleTest(&argc, argv);
506  TestTimeouts::Initialize();
507
508  int batch_limit = kDefaultTestBatchLimit;
509  if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
510    return 1;
511
512  fprintf(stdout,
513          "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
514          "own process. For debugging a test inside a debugger, use the\n"
515          "--gtest_filter=<your_test_name> flag along with\n"
516          "--single-process-tests.\n");
517  fflush(stdout);
518
519  MessageLoopForIO message_loop;
520
521  UnitTestLauncherDelegate delegate(batch_limit);
522  base::TestLauncher launcher(&delegate, default_jobs);
523  bool success = launcher.Run(argc, argv);
524
525  fprintf(stdout,
526          "Tests took %" PRId64 " seconds.\n",
527          (base::TimeTicks::Now() - start_time).InSeconds());
528  fflush(stdout);
529
530  return (success ? 0 : 1);
531}
532
533}  // namespace
534
535int LaunchUnitTests(int argc,
536                    char** argv,
537                    const RunTestSuiteCallback& run_test_suite) {
538  return LaunchUnitTestsInternal(
539      argc, argv, run_test_suite, SysInfo::NumberOfProcessors());
540}
541
542int LaunchUnitTestsSerially(int argc,
543                            char** argv,
544                            const RunTestSuiteCallback& run_test_suite) {
545  return LaunchUnitTestsInternal(argc, argv, run_test_suite, 1);
546}
547
548}  // namespace base
549