test_launcher.cc revision 58e6fbe4ee35d65e14b626c557d37565bf8ad179 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/public/test/test_launcher.h"
6
7#include <string>
8#include <vector>
9
10#include "base/command_line.h"
11#include "base/containers/hash_tables.h"
12#include "base/environment.h"
13#include "base/file_util.h"
14#include "base/files/scoped_temp_dir.h"
15#include "base/logging.h"
16#include "base/memory/linked_ptr.h"
17#include "base/memory/scoped_ptr.h"
18#include "base/strings/string_number_conversions.h"
19#include "base/strings/string_util.h"
20#include "base/strings/utf_string_conversions.h"
21#include "base/test/test_launcher.h"
22#include "base/test/test_suite.h"
23#include "base/test/test_timeouts.h"
24#include "base/time/time.h"
25#include "content/public/app/content_main.h"
26#include "content/public/app/content_main_delegate.h"
27#include "content/public/app/startup_helper_win.h"
28#include "content/public/common/content_switches.h"
29#include "content/public/common/sandbox_init.h"
30#include "content/public/test/browser_test.h"
31#include "net/base/escape.h"
32#include "testing/gtest/include/gtest/gtest.h"
33
34#if defined(OS_WIN)
35#include "base/base_switches.h"
36#include "content/common/sandbox_win.h"
37#include "sandbox/win/src/sandbox_factory.h"
38#include "sandbox/win/src/sandbox_types.h"
39#elif defined(OS_MACOSX)
40#include "base/mac/scoped_nsautorelease_pool.h"
41#endif
42
43namespace content {
44
45namespace {
46
47// Tests with this prefix run before the same test without it, and use the same
48// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
49// that span browser restarts.
50const char kPreTestPrefix[] = "PRE_";
51
52// Manual tests only run when --run-manual is specified. This allows writing
53// tests that don't run automatically but are still in the same test binary.
54// This is useful so that a team that wants to run a few tests doesn't have to
55// add a new binary that must be compiled on all builds.
56const char kManualTestPrefix[] = "MANUAL_";
57
58TestLauncherDelegate* g_launcher_delegate;
59}
60
61namespace {
62
63int DoRunTestInternal(const testing::TestCase* test_case,
64                      const std::string& test_name,
65                      const CommandLine& command_line,
66                      base::TimeDelta default_timeout,
67                      bool* was_timeout) {
68  if (test_case) {
69    std::string pre_test_name = test_name;
70    std::string replace_string = std::string(".") + kPreTestPrefix;
71    ReplaceFirstSubstringAfterOffset(&pre_test_name, 0, ".", replace_string);
72    for (int i = 0; i < test_case->total_test_count(); ++i) {
73      const testing::TestInfo* test_info = test_case->GetTestInfo(i);
74      std::string cur_test_name = test_info->test_case_name();
75      cur_test_name.append(".");
76      cur_test_name.append(test_info->name());
77      if (cur_test_name == pre_test_name) {
78        int exit_code = DoRunTestInternal(test_case,
79                                          pre_test_name,
80                                          command_line,
81                                          default_timeout,
82                                          was_timeout);
83        if (exit_code != 0)
84          return exit_code;
85      }
86    }
87  }
88
89  CommandLine new_cmd_line(command_line);
90
91  // Always enable disabled tests.  This method is not called with disabled
92  // tests unless this flag was specified to the browser test executable.
93  new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
94  new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
95  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
96
97  char* browser_wrapper = getenv("BROWSER_WRAPPER");
98  int exit_code = base::LaunchChildGTestProcess(
99      new_cmd_line,
100      browser_wrapper ? browser_wrapper : std::string(),
101      default_timeout,
102      was_timeout);
103  if (*was_timeout) {
104    LOG(ERROR) << "Test timeout (" << default_timeout.InMilliseconds()
105               << " ms) exceeded for " << test_name;
106  }
107
108  return exit_code;
109}
110
111// Runs test specified by |test_name| in a child process,
112// and returns the exit code.
113int DoRunTest(TestLauncherDelegate* launcher_delegate,
114              const testing::TestCase* test_case,
115              const std::string& test_name,
116              base::TimeDelta default_timeout,
117              bool* was_timeout) {
118  if (was_timeout)
119    *was_timeout = false;
120
121#if defined(OS_MACOSX)
122  // Some of the below method calls will leak objects if there is no
123  // autorelease pool in place.
124  base::mac::ScopedNSAutoreleasePool pool;
125#endif
126
127  base::ScopedTempDir temp_dir;
128  // Create a new data dir and pass it to the child.
129  if (!temp_dir.CreateUniqueTempDir() || !temp_dir.IsValid()) {
130    LOG(ERROR) << "Error creating temp data directory";
131    return -1;
132  }
133
134  CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
135  if (!launcher_delegate->AdjustChildProcessCommandLine(&new_cmd_line,
136                                                        temp_dir.path())) {
137    return -1;
138  }
139
140  return DoRunTestInternal(
141      test_case, test_name, new_cmd_line, default_timeout, was_timeout);
142}
143
144void PrintUsage() {
145  fprintf(stdout,
146      "Runs tests using the gtest framework, each test being run in its own\n"
147      "process.  Any gtest flags can be specified.\n"
148      "  --single_process\n"
149      "    Runs the tests and the launcher in the same process. Useful for \n"
150      "    debugging a specific test in a debugger.\n"
151      "  --single-process\n"
152      "    Same as above, and also runs Chrome in single-process mode.\n"
153      "  --help\n"
154      "    Shows this message.\n"
155      "  --gtest_help\n"
156      "    Shows the gtest help message.\n");
157}
158
159// Implementation of base::TestLauncherDelegate. This is also a test launcher,
160// wrapping a lower-level test launcher with content-specific code.
161class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
162 public:
163  explicit WrapperTestLauncherDelegate(
164      content::TestLauncherDelegate* launcher_delegate)
165      : launcher_delegate_(launcher_delegate),
166        timeout_count_(0),
167        printed_timeout_message_(false) {
168  }
169
170  // base::TestLauncherDelegate:
171  virtual bool ShouldRunTest(const testing::TestCase* test_case,
172                             const testing::TestInfo* test_info) OVERRIDE;
173  virtual void RunTest(
174      const testing::TestCase* test_case,
175      const testing::TestInfo* test_info,
176      const base::TestLauncherDelegate::TestResultCallback& callback) OVERRIDE;
177  virtual void RunRemainingTests() OVERRIDE;
178
179 private:
180  content::TestLauncherDelegate* launcher_delegate_;
181
182  // Number of times a test timeout occurred.
183  size_t timeout_count_;
184
185  // True after a message about too many timeouts has been printed,
186  // to avoid doing it more than once.
187  bool printed_timeout_message_;
188
189  DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
190};
191
192bool WrapperTestLauncherDelegate::ShouldRunTest(
193    const testing::TestCase* test_case,
194    const testing::TestInfo* test_info) {
195  std::string test_name =
196      std::string(test_case->name()) + "." + test_info->name();
197
198  if (StartsWithASCII(test_info->name(), kPreTestPrefix, true))
199    return false;
200
201  if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
202      !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
203    return false;
204  }
205
206  // Stop test execution after too many timeouts.
207  if (timeout_count_ > 5) {
208    if (!printed_timeout_message_) {
209      printed_timeout_message_ = true;
210      printf("Too many timeouts, aborting test\n");
211    }
212    return false;
213  }
214
215  return true;
216}
217
218void WrapperTestLauncherDelegate::RunTest(
219    const testing::TestCase* test_case,
220    const testing::TestInfo* test_info,
221    const base::TestLauncherDelegate::TestResultCallback& callback) {
222  base::TimeTicks start_time = base::TimeTicks::Now();
223  bool was_timeout = false;
224  std::string test_name =
225      std::string(test_case->name()) + "." + test_info->name();
226  int exit_code = DoRunTest(launcher_delegate_,
227                            test_case,
228                            test_name,
229                            TestTimeouts::action_max_timeout(),
230                            &was_timeout);
231  if (was_timeout)
232    timeout_count_++;
233
234  base::TestResult result;
235  result.test_case_name = test_case->name();
236  result.test_name = test_info->name();
237  result.success = (exit_code == 0);
238  result.elapsed_time = (base::TimeTicks::Now() - start_time);
239
240  callback.Run(result);
241}
242
243void WrapperTestLauncherDelegate::RunRemainingTests() {
244  // No need to do anything here, we launch tests synchronously.
245}
246
247}  // namespace
248
249// The following is kept for historical reasons (so people that are used to
250// using it don't get surprised).
251const char kChildProcessFlag[]   = "child";
252
253const char kGTestHelpFlag[]   = "gtest_help";
254
255const char kHelpFlag[]   = "help";
256
257const char kLaunchAsBrowser[] = "as-browser";
258
259// See kManualTestPrefix above.
260const char kRunManualTestsFlag[] = "run-manual";
261
262const char kSingleProcessTestsFlag[]   = "single_process";
263
264
265TestLauncherDelegate::~TestLauncherDelegate() {
266}
267
268bool ShouldRunContentMain() {
269#if defined(OS_WIN) || defined(OS_LINUX)
270  CommandLine* command_line = CommandLine::ForCurrentProcess();
271  return command_line->HasSwitch(switches::kProcessType) ||
272         command_line->HasSwitch(kLaunchAsBrowser);
273#else
274  return false;
275#endif  // defined(OS_WIN) || defined(OS_LINUX)
276}
277
278int RunContentMain(int argc, char** argv,
279                   TestLauncherDelegate* launcher_delegate) {
280#if defined(OS_WIN)
281  sandbox::SandboxInterfaceInfo sandbox_info = {0};
282  InitializeSandboxInfo(&sandbox_info);
283  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
284      launcher_delegate->CreateContentMainDelegate());
285  return ContentMain(GetModuleHandle(NULL),
286                     &sandbox_info,
287                     chrome_main_delegate.get());
288#elif defined(OS_LINUX)
289  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
290      launcher_delegate->CreateContentMainDelegate());
291  return ContentMain(argc, const_cast<const char**>(argv),
292                     chrome_main_delegate.get());
293#endif  // defined(OS_WIN)
294  NOTREACHED();
295  return 0;
296}
297
298int LaunchTests(TestLauncherDelegate* launcher_delegate,
299                int argc,
300                char** argv) {
301  DCHECK(!g_launcher_delegate);
302  g_launcher_delegate = launcher_delegate;
303
304  CommandLine::Init(argc, argv);
305  const CommandLine* command_line = CommandLine::ForCurrentProcess();
306
307  if (command_line->HasSwitch(kHelpFlag)) {
308    PrintUsage();
309    return 0;
310  }
311
312  if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
313      (command_line->HasSwitch(switches::kSingleProcess) &&
314       command_line->HasSwitch(base::kGTestFilterFlag)) ||
315      command_line->HasSwitch(base::kGTestListTestsFlag) ||
316      command_line->HasSwitch(kGTestHelpFlag)) {
317#if defined(OS_WIN)
318    if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
319      sandbox::SandboxInterfaceInfo sandbox_info;
320      InitializeSandboxInfo(&sandbox_info);
321      InitializeSandbox(&sandbox_info);
322    }
323#endif
324    return launcher_delegate->RunTestSuite(argc, argv);
325  }
326
327  if (ShouldRunContentMain())
328    return RunContentMain(argc, argv, launcher_delegate);
329
330  fprintf(stdout,
331      "Starting tests...\n"
332      "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
333      "For debugging a test inside a debugger, use the\n"
334      "--gtest_filter=<your_test_name> flag along with either\n"
335      "--single_process (to run the test in one launcher/browser process) or\n"
336      "--single-process (to do the above, and also run Chrome in single-"
337      "process mode).\n");
338
339  base::AtExitManager at_exit;
340  testing::InitGoogleTest(&argc, argv);
341  TestTimeouts::Initialize();
342
343  WrapperTestLauncherDelegate delegate(launcher_delegate);
344  return base::LaunchTests(&delegate, argc, argv);
345}
346
347TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
348  return g_launcher_delegate;
349}
350
351}  // namespace content
352