test_launcher.cc revision eb525c5499e34cc9c4b825d6d9e75bb07cc06ace 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/public/test/test_launcher.h"
6
7#include <string>
8#include <vector>
9
10#include "base/command_line.h"
11#include "base/containers/hash_tables.h"
12#include "base/environment.h"
13#include "base/file_util.h"
14#include "base/files/scoped_temp_dir.h"
15#include "base/logging.h"
16#include "base/memory/linked_ptr.h"
17#include "base/memory/scoped_ptr.h"
18#include "base/process_util.h"
19#include "base/strings/string_number_conversions.h"
20#include "base/strings/string_util.h"
21#include "base/strings/utf_string_conversions.h"
22#include "base/test/test_launcher.h"
23#include "base/test/test_suite.h"
24#include "base/test/test_timeouts.h"
25#include "base/time/time.h"
26#include "content/public/app/content_main.h"
27#include "content/public/app/content_main_delegate.h"
28#include "content/public/app/startup_helper_win.h"
29#include "content/public/common/content_switches.h"
30#include "content/public/common/sandbox_init.h"
31#include "content/public/test/browser_test.h"
32#include "net/base/escape.h"
33#include "testing/gtest/include/gtest/gtest.h"
34
35#if defined(OS_WIN)
36#include "base/base_switches.h"
37#include "content/common/sandbox_win.h"
38#include "sandbox/win/src/sandbox_factory.h"
39#include "sandbox/win/src/sandbox_types.h"
40#elif defined(OS_MACOSX)
41#include "base/mac/scoped_nsautorelease_pool.h"
42#endif
43
44namespace content {
45
46namespace {
47
48// Tests with this prefix run before the same test without it, and use the same
49// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
50// that span browser restarts.
51const char kPreTestPrefix[] = "PRE_";
52
53// Manual tests only run when --run-manual is specified. This allows writing
54// tests that don't run automatically but are still in the same test binary.
55// This is useful so that a team that wants to run a few tests doesn't have to
56// add a new binary that must be compiled on all builds.
57const char kManualTestPrefix[] = "MANUAL_";
58
59TestLauncherDelegate* g_launcher_delegate;
60}
61
62namespace {
63
64int DoRunTestInternal(const testing::TestCase* test_case,
65                      const std::string& test_name,
66                      const CommandLine& command_line,
67                      base::TimeDelta default_timeout,
68                      bool* was_timeout) {
69  if (test_case) {
70    std::string pre_test_name = test_name;
71    std::string replace_string = std::string(".") + kPreTestPrefix;
72    ReplaceFirstSubstringAfterOffset(&pre_test_name, 0, ".", replace_string);
73    for (int i = 0; i < test_case->total_test_count(); ++i) {
74      const testing::TestInfo* test_info = test_case->GetTestInfo(i);
75      std::string cur_test_name = test_info->test_case_name();
76      cur_test_name.append(".");
77      cur_test_name.append(test_info->name());
78      if (cur_test_name == pre_test_name) {
79        int exit_code = DoRunTestInternal(test_case,
80                                          pre_test_name,
81                                          command_line,
82                                          default_timeout,
83                                          was_timeout);
84        if (exit_code != 0)
85          return exit_code;
86      }
87    }
88  }
89
90  CommandLine new_cmd_line(command_line);
91
92  // Always enable disabled tests.  This method is not called with disabled
93  // tests unless this flag was specified to the browser test executable.
94  new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
95  new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
96  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
97
98  char* browser_wrapper = getenv("BROWSER_WRAPPER");
99  int exit_code = base::LaunchChildGTestProcess(
100      new_cmd_line,
101      browser_wrapper ? browser_wrapper : std::string(),
102      default_timeout,
103      was_timeout);
104  if (*was_timeout) {
105    LOG(ERROR) << "Test timeout (" << default_timeout.InMilliseconds()
106               << " ms) exceeded for " << test_name;
107  }
108
109  return exit_code;
110}
111
112// Runs test specified by |test_name| in a child process,
113// and returns the exit code.
114int DoRunTest(TestLauncherDelegate* launcher_delegate,
115              const testing::TestCase* test_case,
116              const std::string& test_name,
117              base::TimeDelta default_timeout,
118              bool* was_timeout) {
119  if (was_timeout)
120    *was_timeout = false;
121
122#if defined(OS_MACOSX)
123  // Some of the below method calls will leak objects if there is no
124  // autorelease pool in place.
125  base::mac::ScopedNSAutoreleasePool pool;
126#endif
127
128  base::ScopedTempDir temp_dir;
129  // Create a new data dir and pass it to the child.
130  if (!temp_dir.CreateUniqueTempDir() || !temp_dir.IsValid()) {
131    LOG(ERROR) << "Error creating temp data directory";
132    return -1;
133  }
134
135  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
136  if (!launcher_delegate->AdjustChildProcessCommandLine(&new_cmd_line,
137                                                        temp_dir.path())) {
138    return -1;
139  }
140
141  return DoRunTestInternal(
142      test_case, test_name, new_cmd_line, default_timeout, was_timeout);
143}
144
145void PrintUsage() {
146  fprintf(stdout,
147      "Runs tests using the gtest framework, each test being run in its own\n"
148      "process.  Any gtest flags can be specified.\n"
149      "  --single_process\n"
150      "    Runs the tests and the launcher in the same process. Useful for \n"
151      "    debugging a specific test in a debugger.\n"
152      "  --single-process\n"
153      "    Same as above, and also runs Chrome in single-process mode.\n"
154      "  --help\n"
155      "    Shows this message.\n"
156      "  --gtest_help\n"
157      "    Shows the gtest help message.\n");
158}
159
160// Implementation of base::TestLauncherDelegate. This is also a test launcher,
161// wrapping a lower-level test launcher with content-specific code.
162class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
163 public:
164  explicit WrapperTestLauncherDelegate(
165      content::TestLauncherDelegate* launcher_delegate)
166      : launcher_delegate_(launcher_delegate),
167        timeout_count_(0),
168        printed_timeout_message_(false) {
169  }
170
171  // base::TestLauncherDelegate:
172  virtual bool ShouldRunTest(const testing::TestCase* test_case,
173                             const testing::TestInfo* test_info) OVERRIDE;
174  virtual void RunTest(
175      const testing::TestCase* test_case,
176      const testing::TestInfo* test_info,
177      const base::TestLauncherDelegate::TestResultCallback& callback) OVERRIDE;
178  virtual void RunRemainingTests() OVERRIDE;
179
180 private:
181  content::TestLauncherDelegate* launcher_delegate_;
182
183  // Number of times a test timeout occurred.
184  size_t timeout_count_;
185
186  // True after a message about too many timeouts has been printed,
187  // to avoid doing it more than once.
188  bool printed_timeout_message_;
189
190  DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
191};
192
193bool WrapperTestLauncherDelegate::ShouldRunTest(
194    const testing::TestCase* test_case,
195    const testing::TestInfo* test_info) {
196  std::string test_name =
197      std::string(test_case->name()) + "." + test_info->name();
198
199  if (StartsWithASCII(test_info->name(), kPreTestPrefix, true))
200    return false;
201
202  if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
203      !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
204    return false;
205  }
206
207  // Stop test execution after too many timeouts.
208  if (timeout_count_ > 5) {
209    if (!printed_timeout_message_) {
210      printed_timeout_message_ = true;
211      printf("Too many timeouts, aborting test\n");
212    }
213    return false;
214  }
215
216  return true;
217}
218
219void WrapperTestLauncherDelegate::RunTest(
220    const testing::TestCase* test_case,
221    const testing::TestInfo* test_info,
222    const base::TestLauncherDelegate::TestResultCallback& callback) {
223  base::TimeTicks start_time = base::TimeTicks::Now();
224  bool was_timeout = false;
225  std::string test_name =
226      std::string(test_case->name()) + "." + test_info->name();
227  int exit_code = DoRunTest(launcher_delegate_,
228                            test_case,
229                            test_name,
230                            TestTimeouts::action_max_timeout(),
231                            &was_timeout);
232  if (was_timeout)
233    timeout_count_++;
234
235  base::TestResult result;
236  result.test_case_name = test_case->name();
237  result.test_name = test_info->name();
238  result.success = (exit_code == 0);
239  result.elapsed_time = (base::TimeTicks::Now() - start_time);
240
241  callback.Run(result);
242}
243
244void WrapperTestLauncherDelegate::RunRemainingTests() {
245  // No need to do anything here, we launch tests synchronously.
246}
247
248}  // namespace
249
250// The following is kept for historical reasons (so people that are used to
251// using it don't get surprised).
252const char kChildProcessFlag[]   = "child";
253
254const char kGTestHelpFlag[]   = "gtest_help";
255
256const char kHelpFlag[]   = "help";
257
258const char kLaunchAsBrowser[] = "as-browser";
259
260// See kManualTestPrefix above.
261const char kRunManualTestsFlag[] = "run-manual";
262
263const char kSingleProcessTestsFlag[]   = "single_process";
264
265
266TestLauncherDelegate::~TestLauncherDelegate() {
267}
268
269bool ShouldRunContentMain() {
270#if defined(OS_WIN) || defined(OS_LINUX)
271  CommandLine* command_line = CommandLine::ForCurrentProcess();
272  return command_line->HasSwitch(switches::kProcessType) ||
273         command_line->HasSwitch(kLaunchAsBrowser);
274#else
275  return false;
276#endif  // defined(OS_WIN) || defined(OS_LINUX)
277}
278
279int RunContentMain(int argc, char** argv,
280                   TestLauncherDelegate* launcher_delegate) {
281#if defined(OS_WIN)
282  sandbox::SandboxInterfaceInfo sandbox_info = {0};
283  InitializeSandboxInfo(&sandbox_info);
284  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
285      launcher_delegate->CreateContentMainDelegate());
286  return ContentMain(GetModuleHandle(NULL),
287                     &sandbox_info,
288                     chrome_main_delegate.get());
289#elif defined(OS_LINUX)
290  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
291      launcher_delegate->CreateContentMainDelegate());
292  return ContentMain(argc, const_cast<const char**>(argv),
293                     chrome_main_delegate.get());
294#endif  // defined(OS_WIN)
295  NOTREACHED();
296  return 0;
297}
298
299int LaunchTests(TestLauncherDelegate* launcher_delegate,
300                int argc,
301                char** argv) {
302  DCHECK(!g_launcher_delegate);
303  g_launcher_delegate = launcher_delegate;
304
305  CommandLine::Init(argc, argv);
306  const CommandLine* command_line = CommandLine::ForCurrentProcess();
307
308  if (command_line->HasSwitch(kHelpFlag)) {
309    PrintUsage();
310    return 0;
311  }
312
313  if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
314      (command_line->HasSwitch(switches::kSingleProcess) &&
315       command_line->HasSwitch(base::kGTestFilterFlag)) ||
316      command_line->HasSwitch(base::kGTestListTestsFlag) ||
317      command_line->HasSwitch(kGTestHelpFlag)) {
318#if defined(OS_WIN)
319    if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
320      sandbox::SandboxInterfaceInfo sandbox_info;
321      InitializeSandboxInfo(&sandbox_info);
322      InitializeSandbox(&sandbox_info);
323    }
324#endif
325    return launcher_delegate->RunTestSuite(argc, argv);
326  }
327
328  if (ShouldRunContentMain())
329    return RunContentMain(argc, argv, launcher_delegate);
330
331  fprintf(stdout,
332      "Starting tests...\n"
333      "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
334      "For debugging a test inside a debugger, use the\n"
335      "--gtest_filter=<your_test_name> flag along with either\n"
336      "--single_process (to run the test in one launcher/browser process) or\n"
337      "--single-process (to do the above, and also run Chrome in single-"
338      "process mode).\n");
339
340  base::AtExitManager at_exit;
341  testing::InitGoogleTest(&argc, argv);
342  TestTimeouts::Initialize();
343
344  WrapperTestLauncherDelegate delegate(launcher_delegate);
345  return base::LaunchTests(&delegate, argc, argv);
346}
347
348TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
349  return g_launcher_delegate;
350}
351
352}  // namespace content
353