1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/test/test_launcher.h"
6
7#include "base/at_exit.h"
8#include "base/bind.h"
9#include "base/command_line.h"
10#include "base/environment.h"
11#include "base/file_util.h"
12#include "base/files/file_path.h"
13#include "base/format_macros.h"
14#include "base/logging.h"
15#include "base/memory/scoped_ptr.h"
16#include "base/process/kill.h"
17#include "base/process/launch.h"
18#include "base/strings/string_number_conversions.h"
19#include "base/strings/utf_string_conversions.h"
20#include "base/test/test_timeouts.h"
21#include "base/time/time.h"
22#include "testing/gtest/include/gtest/gtest.h"
23
24#if defined(OS_MACOSX)
25#include "base/mac/scoped_nsautorelease_pool.h"
26#endif
27
28namespace base {
29
30// See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
31using ::operator<<;
32
33// The environment variable name for the total number of test shards.
34const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
35// The environment variable name for the test shard index.
36const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
37
38// The default output file for XML output.
39const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
40    "test_detail.xml");
41
42namespace {
43
44// Parses the environment variable var as an Int32.  If it is unset, returns
45// default_val.  If it is set, unsets it then converts it to Int32 before
46// returning it.  If unsetting or converting to an Int32 fails, print an
47// error and exit with failure.
48int32 Int32FromEnvOrDie(const char* const var, int32 default_val) {
49  scoped_ptr<Environment> env(Environment::Create());
50  std::string str_val;
51  int32 result;
52  if (!env->GetVar(var, &str_val))
53    return default_val;
54  if (!env->UnSetVar(var)) {
55    LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
56    exit(EXIT_FAILURE);
57  }
58  if (!StringToInt(str_val, &result)) {
59    LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
60    exit(EXIT_FAILURE);
61  }
62  return result;
63}
64
65// Checks whether sharding is enabled by examining the relevant
66// environment variable values.  If the variables are present,
67// but inconsistent (i.e., shard_index >= total_shards), prints
68// an error and exits.
69void InitSharding(int32* total_shards, int32* shard_index) {
70  *total_shards = Int32FromEnvOrDie(kTestTotalShards, 1);
71  *shard_index = Int32FromEnvOrDie(kTestShardIndex, 0);
72
73  if (*total_shards == -1 && *shard_index != -1) {
74    LOG(ERROR) << "Invalid environment variables: you have "
75               << kTestShardIndex << " = " << *shard_index
76               << ", but have left " << kTestTotalShards << " unset.\n";
77    exit(EXIT_FAILURE);
78  } else if (*total_shards != -1 && *shard_index == -1) {
79    LOG(ERROR) << "Invalid environment variables: you have "
80               << kTestTotalShards << " = " << *total_shards
81               << ", but have left " << kTestShardIndex << " unset.\n";
82    exit(EXIT_FAILURE);
83  } else if (*shard_index < 0 || *shard_index >= *total_shards) {
84    LOG(ERROR) << "Invalid environment variables: we require 0 <= "
85               << kTestShardIndex << " < " << kTestTotalShards
86               << ", but you have " << kTestShardIndex << "=" << *shard_index
87               << ", " << kTestTotalShards << "=" << *total_shards << ".\n";
88    exit(EXIT_FAILURE);
89  }
90}
91
92// Given the total number of shards, the shard index, and the test id, returns
93// true iff the test should be run on this shard.  The test id is some arbitrary
94// but unique non-negative integer assigned by this launcher to each test
95// method.  Assumes that 0 <= shard_index < total_shards, which is first
96// verified in ShouldShard().
97bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) {
98  return (test_id % total_shards) == shard_index;
99}
100
101// A helper class to output results.
102// Note: as currently XML is the only supported format by gtest, we don't
103// check output format (e.g. "xml:" prefix) here and output an XML file
104// unconditionally.
105// Note: we don't output per-test-case or total summary info like
106// total failed_test_count, disabled_test_count, elapsed_time and so on.
107// Only each test (testcase element in the XML) will have the correct
108// failed/disabled/elapsed_time information. Each test won't include
109// detailed failure messages either.
110class ResultsPrinter {
111 public:
112  explicit ResultsPrinter(const CommandLine& command_line);
113  ~ResultsPrinter();
114
115  // Adds |result| to the stored test results.
116  void AddTestResult(const TestResult& result);
117
118  // Returns list of full names of failed tests.
119  const std::vector<std::string>& failed_tests() const { return failed_tests_; }
120
121  // Returns total number of tests run.
122  size_t test_run_count() const { return test_run_count_; }
123
124 private:
125  // Test results grouped by test case name.
126  typedef std::map<std::string, std::vector<TestResult> > ResultsMap;
127  ResultsMap results_;
128
129  // List of full names of failed tests.
130  std::vector<std::string> failed_tests_;
131
132  // Total number of tests run.
133  size_t test_run_count_;
134
135  // File handle of output file (can be NULL if no file).
136  FILE* out_;
137
138  DISALLOW_COPY_AND_ASSIGN(ResultsPrinter);
139};
140
141ResultsPrinter::ResultsPrinter(const CommandLine& command_line)
142    : test_run_count_(0),
143      out_(NULL) {
144  if (!command_line.HasSwitch(kGTestOutputFlag))
145    return;
146  std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
147  size_t colon_pos = flag.find(':');
148  FilePath path;
149  if (colon_pos != std::string::npos) {
150    FilePath flag_path =
151        command_line.GetSwitchValuePath(kGTestOutputFlag);
152    FilePath::StringType path_string = flag_path.value();
153    path = FilePath(path_string.substr(colon_pos + 1));
154    // If the given path ends with '/', consider it is a directory.
155    // Note: This does NOT check that a directory (or file) actually exists
156    // (the behavior is same as what gtest does).
157    if (path.EndsWithSeparator()) {
158      FilePath executable = command_line.GetProgram().BaseName();
159      path = path.Append(executable.ReplaceExtension(
160          FilePath::StringType(FILE_PATH_LITERAL("xml"))));
161    }
162  }
163  if (path.value().empty())
164    path = FilePath(kDefaultOutputFile);
165  FilePath dir_name = path.DirName();
166  if (!DirectoryExists(dir_name)) {
167    LOG(WARNING) << "The output directory does not exist. "
168                 << "Creating the directory: " << dir_name.value();
169    // Create the directory if necessary (because the gtest does the same).
170    file_util::CreateDirectory(dir_name);
171  }
172  out_ = file_util::OpenFile(path, "w");
173  if (!out_) {
174    LOG(ERROR) << "Cannot open output file: "
175               << path.value() << ".";
176    return;
177  }
178}
179
180ResultsPrinter::~ResultsPrinter() {
181  if (!out_)
182    return;
183  fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
184  fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\""
185          " disabled=\"\" errors=\"\" time=\"\">\n");
186  for (ResultsMap::iterator i = results_.begin(); i != results_.end(); ++i) {
187    fprintf(out_, "  <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\""
188            " disabled=\"\" errors=\"\" time=\"\">\n",
189            i->first.c_str(), i->second.size());
190    for (size_t j = 0; j < i->second.size(); ++j) {
191      const TestResult& result = i->second[j];
192      fprintf(out_, "    <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
193              " classname=\"%s\">\n",
194              result.test_name.c_str(),
195              result.elapsed_time.InSecondsF(),
196              result.test_case_name.c_str());
197      if (!result.success)
198        fprintf(out_, "      <failure message=\"\" type=\"\"></failure>\n");
199      fprintf(out_, "    </testcase>\n");
200    }
201    fprintf(out_, "  </testsuite>\n");
202  }
203  fprintf(out_, "</testsuites>\n");
204  fclose(out_);
205}
206
207void ResultsPrinter::AddTestResult(const TestResult& result) {
208  ++test_run_count_;
209  results_[result.test_case_name].push_back(result);
210
211  if (!result.success) {
212    failed_tests_.push_back(
213        std::string(result.test_case_name) + "." + result.test_name);
214  }
215}
216
217// For a basic pattern matching for gtest_filter options.  (Copied from
218// gtest.cc, see the comment below and http://crbug.com/44497)
219bool PatternMatchesString(const char* pattern, const char* str) {
220  switch (*pattern) {
221    case '\0':
222    case ':':  // Either ':' or '\0' marks the end of the pattern.
223      return *str == '\0';
224    case '?':  // Matches any single character.
225      return *str != '\0' && PatternMatchesString(pattern + 1, str + 1);
226    case '*':  // Matches any string (possibly empty) of characters.
227      return (*str != '\0' && PatternMatchesString(pattern, str + 1)) ||
228          PatternMatchesString(pattern + 1, str);
229    default:  // Non-special character.  Matches itself.
230      return *pattern == *str &&
231          PatternMatchesString(pattern + 1, str + 1);
232  }
233}
234
235// TODO(phajdan.jr): Avoid duplicating gtest code. (http://crbug.com/44497)
236// For basic pattern matching for gtest_filter options.  (Copied from
237// gtest.cc)
238bool MatchesFilter(const std::string& name, const std::string& filter) {
239  const char *cur_pattern = filter.c_str();
240  for (;;) {
241    if (PatternMatchesString(cur_pattern, name.c_str())) {
242      return true;
243    }
244
245    // Finds the next pattern in the filter.
246    cur_pattern = strchr(cur_pattern, ':');
247
248    // Returns if no more pattern can be found.
249    if (cur_pattern == NULL) {
250      return false;
251    }
252
253    // Skips the pattern separater (the ':' character).
254    cur_pattern++;
255  }
256}
257
258bool RunTests(TestLauncherDelegate* launcher_delegate,
259              int total_shards,
260              int shard_index) {
261  const CommandLine* command_line = CommandLine::ForCurrentProcess();
262
263  DCHECK(!command_line->HasSwitch(kGTestListTestsFlag));
264
265  testing::UnitTest* const unit_test = testing::UnitTest::GetInstance();
266
267  std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
268
269  // Split --gtest_filter at '-', if there is one, to separate into
270  // positive filter and negative filter portions.
271  std::string positive_filter = filter;
272  std::string negative_filter;
273  size_t dash_pos = filter.find('-');
274  if (dash_pos != std::string::npos) {
275    positive_filter = filter.substr(0, dash_pos);  // Everything up to the dash.
276    negative_filter = filter.substr(dash_pos + 1); // Everything after the dash.
277  }
278
279  int num_runnable_tests = 0;
280
281  ResultsPrinter printer(*command_line);
282  for (int i = 0; i < unit_test->total_test_case_count(); ++i) {
283    const testing::TestCase* test_case = unit_test->GetTestCase(i);
284    for (int j = 0; j < test_case->total_test_count(); ++j) {
285      const testing::TestInfo* test_info = test_case->GetTestInfo(j);
286      std::string test_name = test_info->test_case_name();
287      test_name.append(".");
288      test_name.append(test_info->name());
289
290      // Skip disabled tests.
291      if (test_name.find("DISABLED") != std::string::npos &&
292          !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) {
293        continue;
294      }
295
296      // Skip the test that doesn't match the filter string (if given).
297      if ((!positive_filter.empty() &&
298           !MatchesFilter(test_name, positive_filter)) ||
299          MatchesFilter(test_name, negative_filter)) {
300        continue;
301      }
302
303      if (!launcher_delegate->ShouldRunTest(test_case, test_info))
304        continue;
305
306      bool should_run = ShouldRunTestOnShard(total_shards, shard_index,
307                                             num_runnable_tests);
308      num_runnable_tests += 1;
309      if (!should_run)
310        continue;
311
312      launcher_delegate->RunTest(test_case,
313                                 test_info,
314                                 base::Bind(
315                                     &ResultsPrinter::AddTestResult,
316                                     base::Unretained(&printer)));
317    }
318  }
319
320  launcher_delegate->RunRemainingTests();
321
322  printf("%" PRIuS " test%s run\n",
323         printer.test_run_count(),
324         printer.test_run_count() > 1 ? "s" : "");
325  printf("%" PRIuS " test%s failed\n",
326         printer.failed_tests().size(),
327         printer.failed_tests().size() != 1 ? "s" : "");
328  if (printer.failed_tests().empty())
329    return true;
330
331  printf("Failing tests:\n");
332  for (size_t i = 0; i < printer.failed_tests().size(); ++i)
333    printf("%s\n", printer.failed_tests()[i].c_str());
334
335  return false;
336}
337
338}  // namespace
339
340const char kGTestFilterFlag[] = "gtest_filter";
341const char kGTestListTestsFlag[] = "gtest_list_tests";
342const char kGTestRepeatFlag[] = "gtest_repeat";
343const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
344const char kGTestOutputFlag[] = "gtest_output";
345
346const char kHelpFlag[]   = "help";
347
348TestResult::TestResult() {
349}
350
351TestLauncherDelegate::~TestLauncherDelegate() {
352}
353
354int LaunchChildGTestProcess(const CommandLine& command_line,
355                            const std::string& wrapper,
356                            base::TimeDelta timeout,
357                            bool* was_timeout) {
358  CommandLine new_command_line(command_line.GetProgram());
359  CommandLine::SwitchMap switches = command_line.GetSwitches();
360
361  // Strip out gtest_output flag because otherwise we would overwrite results
362  // of the other tests.
363  switches.erase(kGTestOutputFlag);
364
365  // Strip out gtest_repeat flag - this is handled by the launcher process.
366  switches.erase(kGTestRepeatFlag);
367
368  for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
369       iter != switches.end(); ++iter) {
370    new_command_line.AppendSwitchNative((*iter).first, (*iter).second);
371  }
372
373  // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine
374  // does not really support removing switches well, and trying to do that
375  // on a CommandLine with a wrapper is known to break.
376  // TODO(phajdan.jr): Give it a try to support CommandLine removing switches.
377#if defined(OS_WIN)
378  new_command_line.PrependWrapper(ASCIIToWide(wrapper));
379#elif defined(OS_POSIX)
380  new_command_line.PrependWrapper(wrapper);
381#endif
382
383  base::ProcessHandle process_handle;
384  base::LaunchOptions options;
385
386#if defined(OS_POSIX)
387  // On POSIX, we launch the test in a new process group with pgid equal to
388  // its pid. Any child processes that the test may create will inherit the
389  // same pgid. This way, if the test is abruptly terminated, we can clean up
390  // any orphaned child processes it may have left behind.
391  options.new_process_group = true;
392#endif
393
394  if (!base::LaunchProcess(new_command_line, options, &process_handle))
395    return -1;
396
397  int exit_code = 0;
398  if (!base::WaitForExitCodeWithTimeout(process_handle,
399                                        &exit_code,
400                                        timeout)) {
401    *was_timeout = true;
402    exit_code = -1;  // Set a non-zero exit code to signal a failure.
403
404    // Ensure that the process terminates.
405    base::KillProcess(process_handle, -1, true);
406  }
407
408#if defined(OS_POSIX)
409  if (exit_code != 0) {
410    // On POSIX, in case the test does not exit cleanly, either due to a crash
411    // or due to it timing out, we need to clean up any child processes that
412    // it might have created. On Windows, child processes are automatically
413    // cleaned up using JobObjects.
414    base::KillProcessGroup(process_handle);
415  }
416#endif
417
418  base::CloseProcessHandle(process_handle);
419
420  return exit_code;
421}
422
423int LaunchTests(TestLauncherDelegate* launcher_delegate,
424                int argc,
425                char** argv) {
426  const CommandLine* command_line = CommandLine::ForCurrentProcess();
427
428  int32 total_shards;
429  int32 shard_index;
430  InitSharding(&total_shards, &shard_index);
431
432  int cycles = 1;
433  if (command_line->HasSwitch(kGTestRepeatFlag))
434    StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag), &cycles);
435
436  int exit_code = 0;
437  while (cycles != 0) {
438    if (!RunTests(launcher_delegate, total_shards, shard_index)) {
439      exit_code = 1;
440      break;
441    }
442
443    // Special value "-1" means "repeat indefinitely".
444    if (cycles != -1)
445      cycles--;
446  }
447
448  return exit_code;
449}
450
451}  // namespace base
452