test_launcher.cc revision 3551c9c881056c480085172ff9840cab31610854 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/public/test/test_launcher.h"
6
7#include <string>
8#include <vector>
9
10#include "base/command_line.h"
11#include "base/containers/hash_tables.h"
12#include "base/environment.h"
13#include "base/file_util.h"
14#include "base/files/scoped_temp_dir.h"
15#include "base/logging.h"
16#include "base/memory/linked_ptr.h"
17#include "base/memory/scoped_ptr.h"
18#include "base/strings/string_number_conversions.h"
19#include "base/strings/string_util.h"
20#include "base/strings/utf_string_conversions.h"
21#include "base/test/test_launcher.h"
22#include "base/test/test_suite.h"
23#include "base/test/test_timeouts.h"
24#include "base/time/time.h"
25#include "content/public/app/content_main.h"
26#include "content/public/app/content_main_delegate.h"
27#include "content/public/app/startup_helper_win.h"
28#include "content/public/common/content_switches.h"
29#include "content/public/common/sandbox_init.h"
30#include "content/public/test/browser_test.h"
31#include "net/base/escape.h"
32#include "testing/gtest/include/gtest/gtest.h"
33
34#if defined(OS_WIN)
35#include "base/base_switches.h"
36#include "content/common/sandbox_win.h"
37#include "sandbox/win/src/sandbox_factory.h"
38#include "sandbox/win/src/sandbox_types.h"
39#elif defined(OS_MACOSX)
40#include "base/mac/scoped_nsautorelease_pool.h"
41#endif
42
43namespace content {
44
45namespace {
46
47// Tests with this prefix run before the same test without it, and use the same
48// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
49// that span browser restarts.
50const char kPreTestPrefix[] = "PRE_";
51
52// Manual tests only run when --run-manual is specified. This allows writing
53// tests that don't run automatically but are still in the same test binary.
54// This is useful so that a team that wants to run a few tests doesn't have to
55// add a new binary that must be compiled on all builds.
56const char kManualTestPrefix[] = "MANUAL_";
57
58TestLauncherDelegate* g_launcher_delegate;
59}
60
61namespace {
62
63int DoRunTestInternal(const testing::TestCase* test_case,
64                      const std::string& test_name,
65                      const CommandLine& command_line,
66                      base::TimeDelta default_timeout,
67                      bool* was_timeout) {
68  if (test_case) {
69    std::string pre_test_name = test_name;
70    std::string replace_string = std::string(".") + kPreTestPrefix;
71    ReplaceFirstSubstringAfterOffset(&pre_test_name, 0, ".", replace_string);
72    for (int i = 0; i < test_case->total_test_count(); ++i) {
73      const testing::TestInfo* test_info = test_case->GetTestInfo(i);
74      std::string cur_test_name = test_info->test_case_name();
75      cur_test_name.append(".");
76      cur_test_name.append(test_info->name());
77      if (cur_test_name == pre_test_name) {
78        int exit_code = DoRunTestInternal(test_case,
79                                          pre_test_name,
80                                          command_line,
81                                          default_timeout,
82                                          was_timeout);
83        if (exit_code != 0)
84          return exit_code;
85      }
86    }
87  }
88
89  CommandLine new_cmd_line(command_line.GetProgram());
90  CommandLine::SwitchMap switches = command_line.GetSwitches();
91
92  // Strip out gtest_output flag because otherwise we would overwrite results
93  // of the other tests.
94  switches.erase(base::kGTestOutputFlag);
95
96  for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
97       iter != switches.end(); ++iter) {
98    new_cmd_line.AppendSwitchNative(iter->first, iter->second);
99  }
100
101  // Always enable disabled tests.  This method is not called with disabled
102  // tests unless this flag was specified to the browser test executable.
103  new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
104  new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
105  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
106
107  char* browser_wrapper = getenv("BROWSER_WRAPPER");
108  int exit_code = base::LaunchChildGTestProcess(
109      new_cmd_line,
110      browser_wrapper ? browser_wrapper : std::string(),
111      default_timeout,
112      was_timeout);
113  if (*was_timeout) {
114    LOG(ERROR) << "Test timeout (" << default_timeout.InMilliseconds()
115               << " ms) exceeded for " << test_name;
116  }
117
118  return exit_code;
119}
120
121// Runs test specified by |test_name| in a child process,
122// and returns the exit code.
123int DoRunTest(TestLauncherDelegate* launcher_delegate,
124              const testing::TestCase* test_case,
125              const std::string& test_name,
126              base::TimeDelta default_timeout,
127              bool* was_timeout) {
128  if (was_timeout)
129    *was_timeout = false;
130
131#if defined(OS_MACOSX)
132  // Some of the below method calls will leak objects if there is no
133  // autorelease pool in place.
134  base::mac::ScopedNSAutoreleasePool pool;
135#endif
136
137  base::ScopedTempDir temp_dir;
138  // Create a new data dir and pass it to the child.
139  if (!temp_dir.CreateUniqueTempDir() || !temp_dir.IsValid()) {
140    LOG(ERROR) << "Error creating temp data directory";
141    return -1;
142  }
143
144  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
145  if (!launcher_delegate->AdjustChildProcessCommandLine(&new_cmd_line,
146                                                        temp_dir.path())) {
147    return -1;
148  }
149
150  return DoRunTestInternal(
151      test_case, test_name, new_cmd_line, default_timeout, was_timeout);
152}
153
154void PrintUsage() {
155  fprintf(stdout,
156      "Runs tests using the gtest framework, each test being run in its own\n"
157      "process.  Any gtest flags can be specified.\n"
158      "  --single_process\n"
159      "    Runs the tests and the launcher in the same process. Useful for \n"
160      "    debugging a specific test in a debugger.\n"
161      "  --single-process\n"
162      "    Same as above, and also runs Chrome in single-process mode.\n"
163      "  --help\n"
164      "    Shows this message.\n"
165      "  --gtest_help\n"
166      "    Shows the gtest help message.\n");
167}
168
169// Implementation of base::TestLauncherDelegate. This is also a test launcher,
170// wrapping a lower-level test launcher with content-specific code.
171class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
172 public:
173  explicit WrapperTestLauncherDelegate(
174      content::TestLauncherDelegate* launcher_delegate)
175      : launcher_delegate_(launcher_delegate),
176        timeout_count_(0),
177        printed_timeout_message_(false) {
178  }
179
180  // base::TestLauncherDelegate:
181  virtual bool ShouldRunTest(const testing::TestCase* test_case,
182                             const testing::TestInfo* test_info) OVERRIDE;
183  virtual void RunTest(
184      const testing::TestCase* test_case,
185      const testing::TestInfo* test_info,
186      const base::TestLauncherDelegate::TestResultCallback& callback) OVERRIDE;
187  virtual void RunRemainingTests() OVERRIDE;
188
189 private:
190  content::TestLauncherDelegate* launcher_delegate_;
191
192  // Number of times a test timeout occurred.
193  size_t timeout_count_;
194
195  // True after a message about too many timeouts has been printed,
196  // to avoid doing it more than once.
197  bool printed_timeout_message_;
198
199  DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
200};
201
202bool WrapperTestLauncherDelegate::ShouldRunTest(
203    const testing::TestCase* test_case,
204    const testing::TestInfo* test_info) {
205  std::string test_name =
206      std::string(test_case->name()) + "." + test_info->name();
207
208  if (StartsWithASCII(test_info->name(), kPreTestPrefix, true))
209    return false;
210
211  if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
212      !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
213    return false;
214  }
215
216  // Stop test execution after too many timeouts.
217  if (timeout_count_ > 5) {
218    if (!printed_timeout_message_) {
219      printed_timeout_message_ = true;
220      printf("Too many timeouts, aborting test\n");
221    }
222    return false;
223  }
224
225  return true;
226}
227
228void WrapperTestLauncherDelegate::RunTest(
229    const testing::TestCase* test_case,
230    const testing::TestInfo* test_info,
231    const base::TestLauncherDelegate::TestResultCallback& callback) {
232  base::TimeTicks start_time = base::TimeTicks::Now();
233  bool was_timeout = false;
234  std::string test_name =
235      std::string(test_case->name()) + "." + test_info->name();
236  int exit_code = DoRunTest(launcher_delegate_,
237                            test_case,
238                            test_name,
239                            TestTimeouts::action_max_timeout(),
240                            &was_timeout);
241  if (was_timeout)
242    timeout_count_++;
243
244  base::TestResult result;
245  result.test_case_name = test_case->name();
246  result.test_name = test_info->name();
247  result.success = (exit_code == 0);
248  result.elapsed_time = (base::TimeTicks::Now() - start_time);
249
250  callback.Run(result);
251}
252
253void WrapperTestLauncherDelegate::RunRemainingTests() {
254  // No need to do anything here, we launch tests synchronously.
255}
256
257}  // namespace
258
259// The following is kept for historical reasons (so people that are used to
260// using it don't get surprised).
261const char kChildProcessFlag[]   = "child";
262
263const char kGTestHelpFlag[]   = "gtest_help";
264
265const char kHelpFlag[]   = "help";
266
267const char kLaunchAsBrowser[] = "as-browser";
268
269// See kManualTestPrefix above.
270const char kRunManualTestsFlag[] = "run-manual";
271
272const char kSingleProcessTestsFlag[]   = "single_process";
273
274
275TestLauncherDelegate::~TestLauncherDelegate() {
276}
277
278bool ShouldRunContentMain() {
279#if defined(OS_WIN) || defined(OS_LINUX)
280  CommandLine* command_line = CommandLine::ForCurrentProcess();
281  return command_line->HasSwitch(switches::kProcessType) ||
282         command_line->HasSwitch(kLaunchAsBrowser);
283#else
284  return false;
285#endif  // defined(OS_WIN) || defined(OS_LINUX)
286}
287
288int RunContentMain(int argc, char** argv,
289                   TestLauncherDelegate* launcher_delegate) {
290#if defined(OS_WIN)
291  sandbox::SandboxInterfaceInfo sandbox_info = {0};
292  InitializeSandboxInfo(&sandbox_info);
293  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
294      launcher_delegate->CreateContentMainDelegate());
295  return ContentMain(GetModuleHandle(NULL),
296                     &sandbox_info,
297                     chrome_main_delegate.get());
298#elif defined(OS_LINUX)
299  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
300      launcher_delegate->CreateContentMainDelegate());
301  return ContentMain(argc, const_cast<const char**>(argv),
302                     chrome_main_delegate.get());
303#endif  // defined(OS_WIN)
304  NOTREACHED();
305  return 0;
306}
307
308int LaunchTests(TestLauncherDelegate* launcher_delegate,
309                int argc,
310                char** argv) {
311  DCHECK(!g_launcher_delegate);
312  g_launcher_delegate = launcher_delegate;
313
314  CommandLine::Init(argc, argv);
315  const CommandLine* command_line = CommandLine::ForCurrentProcess();
316
317  if (command_line->HasSwitch(kHelpFlag)) {
318    PrintUsage();
319    return 0;
320  }
321
322  if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
323      (command_line->HasSwitch(switches::kSingleProcess) &&
324       command_line->HasSwitch(base::kGTestFilterFlag)) ||
325      command_line->HasSwitch(base::kGTestListTestsFlag) ||
326      command_line->HasSwitch(kGTestHelpFlag)) {
327#if defined(OS_WIN)
328    if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
329      sandbox::SandboxInterfaceInfo sandbox_info;
330      InitializeSandboxInfo(&sandbox_info);
331      InitializeSandbox(&sandbox_info);
332    }
333#endif
334    return launcher_delegate->RunTestSuite(argc, argv);
335  }
336
337  if (ShouldRunContentMain())
338    return RunContentMain(argc, argv, launcher_delegate);
339
340  fprintf(stdout,
341      "Starting tests...\n"
342      "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
343      "For debugging a test inside a debugger, use the\n"
344      "--gtest_filter=<your_test_name> flag along with either\n"
345      "--single_process (to run the test in one launcher/browser process) or\n"
346      "--single-process (to do the above, and also run Chrome in single-"
347      "process mode).\n");
348
349  base::AtExitManager at_exit;
350  testing::InitGoogleTest(&argc, argv);
351  TestTimeouts::Initialize();
352
353  WrapperTestLauncherDelegate delegate(launcher_delegate);
354  return base::LaunchTests(&delegate, argc, argv);
355}
356
357TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
358  return g_launcher_delegate;
359}
360
361}  // namespace content
362