1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/test/launcher/unit_test_launcher.h"
6
7#include "base/bind.h"
8#include "base/callback_helpers.h"
9#include "base/command_line.h"
10#include "base/compiler_specific.h"
11#include "base/debug/debugger.h"
12#include "base/files/file_util.h"
13#include "base/files/scoped_temp_dir.h"
14#include "base/format_macros.h"
15#include "base/message_loop/message_loop.h"
16#include "base/stl_util.h"
17#include "base/strings/string_number_conversions.h"
18#include "base/strings/string_util.h"
19#include "base/sys_info.h"
20#include "base/test/gtest_xml_util.h"
21#include "base/test/launcher/test_launcher.h"
22#include "base/test/test_switches.h"
23#include "base/test/test_timeouts.h"
24#include "base/third_party/dynamic_annotations/dynamic_annotations.h"
25#include "base/threading/thread_checker.h"
26#include "testing/gtest/include/gtest/gtest.h"
27
28namespace base {
29
30namespace {
31
32// This constant controls how many tests are run in a single batch by default.
33const size_t kDefaultTestBatchLimit = 10;
34
35const char kHelpFlag[] = "help";
36
37// Flag to run all tests in a single process.
38const char kSingleProcessTestsFlag[] = "single-process-tests";
39
40void PrintUsage() {
41  fprintf(stdout,
42          "Runs tests using the gtest framework, each batch of tests being\n"
43          "run in their own process. Supported command-line flags:\n"
44          "\n"
45          " Common flags:\n"
46          "  --gtest_filter=...\n"
47          "    Runs a subset of tests (see --gtest_help for more info).\n"
48          "\n"
49          "  --help\n"
50          "    Shows this message.\n"
51          "\n"
52          "  --gtest_help\n"
53          "    Shows the gtest help message.\n"
54          "\n"
55          "  --test-launcher-jobs=N\n"
56          "    Sets the number of parallel test jobs to N.\n"
57          "\n"
58          "  --single-process-tests\n"
59          "    Runs the tests and the launcher in the same process. Useful\n"
60          "    for debugging a specific test in a debugger.\n"
61          "\n"
62          " Other flags:\n"
63          "  --test-launcher-batch-limit=N\n"
64          "    Sets the limit of test batch to run in a single process to N.\n"
65          "\n"
66          "  --test-launcher-debug-launcher\n"
67          "    Disables autodetection of debuggers and similar tools,\n"
68          "    making it possible to use them to debug launcher itself.\n"
69          "\n"
70          "  --test-launcher-retry-limit=N\n"
71          "    Sets the limit of test retries on failures to N.\n"
72          "\n"
73          "  --test-launcher-summary-output=PATH\n"
74          "    Saves a JSON machine-readable summary of the run.\n"
75          "\n"
76          "  --test-launcher-print-test-stdio=auto|always|never\n"
77          "    Controls when full test output is printed.\n"
78          "    auto means to print it when the test failed.\n"
79          "\n"
80          "  --test-launcher-total-shards=N\n"
81          "    Sets the total number of shards to N.\n"
82          "\n"
83          "  --test-launcher-shard-index=N\n"
84          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
85  fflush(stdout);
86}
87
88// Returns command line for child GTest process based on the command line
89// of current process. |test_names| is a vector of test full names
90// (e.g. "A.B"), |output_file| is path to the GTest XML output file.
91CommandLine GetCommandLineForChildGTestProcess(
92    const std::vector<std::string>& test_names,
93    const base::FilePath& output_file) {
94  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
95
96  new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
97  new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":"));
98  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
99
100  return new_cmd_line;
101}
102
103class UnitTestLauncherDelegate : public TestLauncherDelegate {
104 public:
105  explicit UnitTestLauncherDelegate(size_t batch_limit, bool use_job_objects)
106      : batch_limit_(batch_limit),
107        use_job_objects_(use_job_objects) {
108  }
109
110  virtual ~UnitTestLauncherDelegate() {
111    DCHECK(thread_checker_.CalledOnValidThread());
112  }
113
114 private:
115  struct GTestCallbackState {
116    TestLauncher* test_launcher;
117    std::vector<std::string> test_names;
118    FilePath output_file;
119  };
120
121  virtual bool ShouldRunTest(const testing::TestCase* test_case,
122                             const testing::TestInfo* test_info) OVERRIDE {
123    DCHECK(thread_checker_.CalledOnValidThread());
124
125    // There is no additional logic to disable specific tests.
126    return true;
127  }
128
129  virtual size_t RunTests(TestLauncher* test_launcher,
130                          const std::vector<std::string>& test_names) OVERRIDE {
131    DCHECK(thread_checker_.CalledOnValidThread());
132
133    std::vector<std::string> batch;
134    for (size_t i = 0; i < test_names.size(); i++) {
135      batch.push_back(test_names[i]);
136
137      if (batch.size() >= batch_limit_) {
138        RunBatch(test_launcher, batch);
139        batch.clear();
140      }
141    }
142
143    RunBatch(test_launcher, batch);
144
145    return test_names.size();
146  }
147
148  virtual size_t RetryTests(
149      TestLauncher* test_launcher,
150      const std::vector<std::string>& test_names) OVERRIDE {
151    MessageLoop::current()->PostTask(
152        FROM_HERE,
153        Bind(&UnitTestLauncherDelegate::RunSerially,
154             Unretained(this),
155             test_launcher,
156             test_names));
157    return test_names.size();
158  }
159
160  void RunSerially(TestLauncher* test_launcher,
161                   const std::vector<std::string>& test_names) {
162    if (test_names.empty())
163      return;
164
165    std::vector<std::string> new_test_names(test_names);
166    std::string test_name(new_test_names.back());
167    new_test_names.pop_back();
168
169    // Create a dedicated temporary directory to store the xml result data
170    // per run to ensure clean state and make it possible to launch multiple
171    // processes in parallel.
172    base::FilePath output_file;
173    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
174    output_file = output_file.AppendASCII("test_results.xml");
175
176    std::vector<std::string> current_test_names;
177    current_test_names.push_back(test_name);
178    CommandLine cmd_line(
179        GetCommandLineForChildGTestProcess(current_test_names, output_file));
180
181    GTestCallbackState callback_state;
182    callback_state.test_launcher = test_launcher;
183    callback_state.test_names = current_test_names;
184    callback_state.output_file = output_file;
185
186    test_launcher->LaunchChildGTestProcess(
187        cmd_line,
188        std::string(),
189        TestTimeouts::test_launcher_timeout(),
190        use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0,
191        Bind(&UnitTestLauncherDelegate::SerialGTestCallback,
192             Unretained(this),
193             callback_state,
194             new_test_names));
195  }
196
197  void RunBatch(TestLauncher* test_launcher,
198                const std::vector<std::string>& test_names) {
199    DCHECK(thread_checker_.CalledOnValidThread());
200
201    if (test_names.empty())
202      return;
203
204    // Create a dedicated temporary directory to store the xml result data
205    // per run to ensure clean state and make it possible to launch multiple
206    // processes in parallel.
207    base::FilePath output_file;
208    CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file));
209    output_file = output_file.AppendASCII("test_results.xml");
210
211    CommandLine cmd_line(
212        GetCommandLineForChildGTestProcess(test_names, output_file));
213
214    // Adjust the timeout depending on how many tests we're running
215    // (note that e.g. the last batch of tests will be smaller).
216    // TODO(phajdan.jr): Consider an adaptive timeout, which can change
217    // depending on how many tests ran and how many remain.
218    // Note: do NOT parse child's stdout to do that, it's known to be
219    // unreliable (e.g. buffering issues can mix up the output).
220    base::TimeDelta timeout =
221        test_names.size() * TestTimeouts::test_launcher_timeout();
222
223    GTestCallbackState callback_state;
224    callback_state.test_launcher = test_launcher;
225    callback_state.test_names = test_names;
226    callback_state.output_file = output_file;
227
228    test_launcher->LaunchChildGTestProcess(
229        cmd_line,
230        std::string(),
231        timeout,
232        use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0,
233        Bind(&UnitTestLauncherDelegate::GTestCallback,
234             Unretained(this),
235             callback_state));
236  }
237
238  void GTestCallback(const GTestCallbackState& callback_state,
239                     int exit_code,
240                     const TimeDelta& elapsed_time,
241                     bool was_timeout,
242                     const std::string& output) {
243    DCHECK(thread_checker_.CalledOnValidThread());
244    std::vector<std::string> tests_to_relaunch;
245    ProcessTestResults(callback_state.test_launcher,
246                       callback_state.test_names,
247                       callback_state.output_file,
248                       output,
249                       exit_code,
250                       was_timeout,
251                       &tests_to_relaunch);
252
253    // Relaunch requested tests in parallel, but only use single
254    // test per batch for more precise results (crashes, test passes
255    // but non-zero exit codes etc).
256    for (size_t i = 0; i < tests_to_relaunch.size(); i++) {
257      std::vector<std::string> batch;
258      batch.push_back(tests_to_relaunch[i]);
259      RunBatch(callback_state.test_launcher, batch);
260    }
261
262    // The temporary file's directory is also temporary.
263    DeleteFile(callback_state.output_file.DirName(), true);
264  }
265
266  void SerialGTestCallback(const GTestCallbackState& callback_state,
267                           const std::vector<std::string>& test_names,
268                           int exit_code,
269                           const TimeDelta& elapsed_time,
270                           bool was_timeout,
271                           const std::string& output) {
272    DCHECK(thread_checker_.CalledOnValidThread());
273    std::vector<std::string> tests_to_relaunch;
274    bool called_any_callbacks =
275        ProcessTestResults(callback_state.test_launcher,
276                           callback_state.test_names,
277                           callback_state.output_file,
278                           output,
279                           exit_code,
280                           was_timeout,
281                           &tests_to_relaunch);
282
283    // There is only one test, there cannot be other tests to relaunch
284    // due to a crash.
285    DCHECK(tests_to_relaunch.empty());
286
287    // There is only one test, we should have called back with its result.
288    DCHECK(called_any_callbacks);
289
290    // The temporary file's directory is also temporary.
291    DeleteFile(callback_state.output_file.DirName(), true);
292
293    MessageLoop::current()->PostTask(
294        FROM_HERE,
295        Bind(&UnitTestLauncherDelegate::RunSerially,
296             Unretained(this),
297             callback_state.test_launcher,
298             test_names));
299  }
300
301  static bool ProcessTestResults(
302      TestLauncher* test_launcher,
303      const std::vector<std::string>& test_names,
304      const base::FilePath& output_file,
305      const std::string& output,
306      int exit_code,
307      bool was_timeout,
308      std::vector<std::string>* tests_to_relaunch) {
309    std::vector<TestResult> test_results;
310    bool crashed = false;
311    bool have_test_results =
312        ProcessGTestOutput(output_file, &test_results, &crashed);
313
314    bool called_any_callback = false;
315
316    if (have_test_results) {
317      // TODO(phajdan.jr): Check for duplicates and mismatches between
318      // the results we got from XML file and tests we intended to run.
319      std::map<std::string, TestResult> results_map;
320      for (size_t i = 0; i < test_results.size(); i++)
321        results_map[test_results[i].full_name] = test_results[i];
322
323      bool had_interrupted_test = false;
324
325      // Results to be reported back to the test launcher.
326      std::vector<TestResult> final_results;
327
328      for (size_t i = 0; i < test_names.size(); i++) {
329        if (ContainsKey(results_map, test_names[i])) {
330          TestResult test_result = results_map[test_names[i]];
331          if (test_result.status == TestResult::TEST_CRASH) {
332            had_interrupted_test = true;
333
334            if (was_timeout) {
335              // Fix up the test status: we forcibly kill the child process
336              // after the timeout, so from XML results it looks just like
337              // a crash.
338              test_result.status = TestResult::TEST_TIMEOUT;
339            }
340          } else if (test_result.status == TestResult::TEST_SUCCESS ||
341                     test_result.status == TestResult::TEST_FAILURE) {
342            // We run multiple tests in a batch with a timeout applied
343            // to the entire batch. It is possible that with other tests
344            // running quickly some tests take longer than the per-test timeout.
345            // For consistent handling of tests independent of order and other
346            // factors, mark them as timing out.
347            if (test_result.elapsed_time >
348                TestTimeouts::test_launcher_timeout()) {
349              test_result.status = TestResult::TEST_TIMEOUT;
350            }
351          }
352          test_result.output_snippet =
353              GetTestOutputSnippet(test_result, output);
354          final_results.push_back(test_result);
355        } else if (had_interrupted_test) {
356          tests_to_relaunch->push_back(test_names[i]);
357        } else {
358          // TODO(phajdan.jr): Explicitly pass the info that the test didn't
359          // run for a mysterious reason.
360          LOG(ERROR) << "no test result for " << test_names[i];
361          TestResult test_result;
362          test_result.full_name = test_names[i];
363          test_result.status = TestResult::TEST_UNKNOWN;
364          test_result.output_snippet =
365              GetTestOutputSnippet(test_result, output);
366          final_results.push_back(test_result);
367        }
368      }
369
370      // TODO(phajdan.jr): Handle the case where processing XML output
371      // indicates a crash but none of the test results is marked as crashing.
372
373      if (final_results.empty())
374        return false;
375
376      bool has_non_success_test = false;
377      for (size_t i = 0; i < final_results.size(); i++) {
378        if (final_results[i].status != TestResult::TEST_SUCCESS) {
379          has_non_success_test = true;
380          break;
381        }
382      }
383
384      if (!has_non_success_test && exit_code != 0) {
385        // This is a bit surprising case: all tests are marked as successful,
386        // but the exit code was not zero. This can happen e.g. under memory
387        // tools that report leaks this way.
388
389        if (final_results.size() == 1) {
390          // Easy case. One test only so we know the non-zero exit code
391          // was caused by that one test.
392          final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT;
393        } else {
394          // Harder case. Discard the results and request relaunching all
395          // tests without batching. This will trigger above branch on
396          // relaunch leading to more precise results.
397          LOG(WARNING) << "Not sure which test caused non-zero exit code, "
398                       << "relaunching all of them without batching.";
399
400          for (size_t i = 0; i < final_results.size(); i++)
401            tests_to_relaunch->push_back(final_results[i].full_name);
402
403          return false;
404        }
405      }
406
407      for (size_t i = 0; i < final_results.size(); i++) {
408        // Fix the output snippet after possible changes to the test result.
409        final_results[i].output_snippet =
410            GetTestOutputSnippet(final_results[i], output);
411        test_launcher->OnTestFinished(final_results[i]);
412        called_any_callback = true;
413      }
414    } else {
415      fprintf(stdout,
416              "Failed to get out-of-band test success data, "
417              "dumping full stdio below:\n%s\n",
418              output.c_str());
419      fflush(stdout);
420
421      // We do not have reliable details about test results (parsing test
422      // stdout is known to be unreliable), apply the executable exit code
423      // to all tests.
424      // TODO(phajdan.jr): Be smarter about this, e.g. retry each test
425      // individually.
426      for (size_t i = 0; i < test_names.size(); i++) {
427        TestResult test_result;
428        test_result.full_name = test_names[i];
429        test_result.status = TestResult::TEST_UNKNOWN;
430        test_launcher->OnTestFinished(test_result);
431        called_any_callback = true;
432      }
433    }
434
435    return called_any_callback;
436  }
437
438  ThreadChecker thread_checker_;
439
440  // Maximum number of tests to run in a single batch.
441  size_t batch_limit_;
442
443  // Determines whether we use job objects on Windows.
444  bool use_job_objects_;
445};
446
447bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
448  if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
449    return true;
450
451  std::string switch_value =
452      CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
453  if (!StringToInt(switch_value, result) || *result < 1) {
454    LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
455    return false;
456  }
457
458  return true;
459}
460
461int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite,
462                            int default_jobs,
463                            bool use_job_objects,
464                            const Closure& gtest_init) {
465#if defined(OS_ANDROID)
466  // We can't easily fork on Android, just run the test suite directly.
467  return run_test_suite.Run();
468#else
469  bool force_single_process = false;
470  if (CommandLine::ForCurrentProcess()->HasSwitch(
471          switches::kTestLauncherDebugLauncher)) {
472    fprintf(stdout, "Forcing test launcher debugging mode.\n");
473    fflush(stdout);
474  } else {
475    if (base::debug::BeingDebugged()) {
476      fprintf(stdout,
477              "Debugger detected, switching to single process mode.\n"
478              "Pass --test-launcher-debug-launcher to debug the launcher "
479              "itself.\n");
480      fflush(stdout);
481      force_single_process = true;
482    }
483
484    if (RunningOnValgrind()) {
485      fprintf(stdout,
486              "Valgrind detected, switching to single process mode.\n"
487              "Pass --test-launcher-debug-launcher to valgrind the launcher "
488              "itself.\n");
489      fflush(stdout);
490      force_single_process = true;
491    }
492  }
493
494  if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
495      CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
496      CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) ||
497      force_single_process) {
498    return run_test_suite.Run();
499  }
500#endif
501
502  if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) {
503    PrintUsage();
504    return 0;
505  }
506
507  base::TimeTicks start_time(base::TimeTicks::Now());
508
509  gtest_init.Run();
510  TestTimeouts::Initialize();
511
512  int batch_limit = kDefaultTestBatchLimit;
513  if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
514    return 1;
515
516  fprintf(stdout,
517          "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
518          "own process. For debugging a test inside a debugger, use the\n"
519          "--gtest_filter=<your_test_name> flag along with\n"
520          "--single-process-tests.\n");
521  fflush(stdout);
522
523  MessageLoopForIO message_loop;
524
525  UnitTestLauncherDelegate delegate(batch_limit, use_job_objects);
526  base::TestLauncher launcher(&delegate, default_jobs);
527  bool success = launcher.Run();
528
529  fprintf(stdout,
530          "Tests took %" PRId64 " seconds.\n",
531          (base::TimeTicks::Now() - start_time).InSeconds());
532  fflush(stdout);
533
534  return (success ? 0 : 1);
535}
536
537void InitGoogleTestChar(int* argc, char** argv) {
538  testing::InitGoogleTest(argc, argv);
539}
540
541#if defined(OS_WIN)
542void InitGoogleTestWChar(int* argc, wchar_t** argv) {
543  testing::InitGoogleTest(argc, argv);
544}
545#endif  // defined(OS_WIN)
546
547}  // namespace
548
549int LaunchUnitTests(int argc,
550                    char** argv,
551                    const RunTestSuiteCallback& run_test_suite) {
552  CommandLine::Init(argc, argv);
553  return LaunchUnitTestsInternal(
554      run_test_suite,
555      SysInfo::NumberOfProcessors(),
556      true,
557      Bind(&InitGoogleTestChar, &argc, argv));
558}
559
560int LaunchUnitTestsSerially(int argc,
561                            char** argv,
562                            const RunTestSuiteCallback& run_test_suite) {
563  CommandLine::Init(argc, argv);
564  return LaunchUnitTestsInternal(
565      run_test_suite,
566      1,
567      true,
568      Bind(&InitGoogleTestChar, &argc, argv));
569}
570
571#if defined(OS_WIN)
572int LaunchUnitTests(int argc,
573                    wchar_t** argv,
574                    bool use_job_objects,
575                    const RunTestSuiteCallback& run_test_suite) {
576  // Windows CommandLine::Init ignores argv anyway.
577  CommandLine::Init(argc, NULL);
578  return LaunchUnitTestsInternal(
579      run_test_suite,
580      SysInfo::NumberOfProcessors(),
581      use_job_objects,
582      Bind(&InitGoogleTestWChar, &argc, argv));
583}
584#endif  // defined(OS_WIN)
585
586}  // namespace base
587