1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/public/test/test_launcher.h"
6
7#include <map>
8#include <string>
9#include <vector>
10
11#include "base/command_line.h"
12#include "base/containers/hash_tables.h"
13#include "base/environment.h"
14#include "base/file_util.h"
15#include "base/files/scoped_temp_dir.h"
16#include "base/logging.h"
17#include "base/memory/linked_ptr.h"
18#include "base/memory/scoped_ptr.h"
19#include "base/message_loop/message_loop.h"
20#include "base/stl_util.h"
21#include "base/strings/string_number_conversions.h"
22#include "base/strings/string_util.h"
23#include "base/strings/utf_string_conversions.h"
24#include "base/test/launcher/test_launcher.h"
25#include "base/test/test_suite.h"
26#include "base/test/test_switches.h"
27#include "base/test/test_timeouts.h"
28#include "base/time/time.h"
29#include "content/public/app/content_main.h"
30#include "content/public/app/content_main_delegate.h"
31#include "content/public/app/startup_helper_win.h"
32#include "content/public/common/content_switches.h"
33#include "content/public/common/sandbox_init.h"
34#include "content/public/test/browser_test.h"
35#include "net/base/escape.h"
36#include "testing/gtest/include/gtest/gtest.h"
37
38#if defined(OS_WIN)
39#include "base/base_switches.h"
40#include "content/common/sandbox_win.h"
41#include "sandbox/win/src/sandbox_factory.h"
42#include "sandbox/win/src/sandbox_types.h"
43#elif defined(OS_MACOSX)
44#include "base/mac/scoped_nsautorelease_pool.h"
45#endif
46
47namespace content {
48
49namespace {
50
51// Tests with this prefix run before the same test without it, and use the same
52// profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
53// that span browser restarts.
54const char kPreTestPrefix[] = "PRE_";
55
56// Manual tests only run when --run-manual is specified. This allows writing
57// tests that don't run automatically but are still in the same test binary.
58// This is useful so that a team that wants to run a few tests doesn't have to
59// add a new binary that must be compiled on all builds.
60const char kManualTestPrefix[] = "MANUAL_";
61
62TestLauncherDelegate* g_launcher_delegate;
63
64std::string RemoveAnyPrePrefixes(const std::string& test_name) {
65  std::string result(test_name);
66  ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string());
67  return result;
68}
69
70void PrintUsage() {
71  fprintf(stdout,
72          "Runs tests using the gtest framework, each batch of tests being\n"
73          "run in their own process. Supported command-line flags:\n"
74          "\n"
75          " Common flags:\n"
76          "  --gtest_filter=...\n"
77          "    Runs a subset of tests (see --gtest_help for more info).\n"
78          "\n"
79          "  --help\n"
80          "    Shows this message.\n"
81          "\n"
82          "  --gtest_help\n"
83          "    Shows the gtest help message.\n"
84          "\n"
85          "  --test-launcher-jobs=N\n"
86          "    Sets the number of parallel test jobs to N.\n"
87          "\n"
88          "  --single_process\n"
89          "    Runs the tests and the launcher in the same process. Useful\n"
90          "    for debugging a specific test in a debugger.\n"
91          "\n"
92          " Other flags:\n"
93          "  --test-launcher-retry-limit=N\n"
94          "    Sets the limit of test retries on failures to N.\n"
95          "\n"
96          "  --test-launcher-summary-output=PATH\n"
97          "    Saves a JSON machine-readable summary of the run.\n"
98          "\n"
99          "  --test-launcher-print-test-stdio=auto|always|never\n"
100          "    Controls when full test output is printed.\n"
101          "    auto means to print it when the test failed.\n"
102          "\n"
103          "  --test-launcher-total-shards=N\n"
104          "    Sets the total number of shards to N.\n"
105          "\n"
106          "  --test-launcher-shard-index=N\n"
107          "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
108}
109
110// Implementation of base::TestLauncherDelegate. This is also a test launcher,
111// wrapping a lower-level test launcher with content-specific code.
112class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
113 public:
114  explicit WrapperTestLauncherDelegate(
115      content::TestLauncherDelegate* launcher_delegate)
116      : launcher_delegate_(launcher_delegate) {
117    CHECK(temp_dir_.CreateUniqueTempDir());
118  }
119
120  // base::TestLauncherDelegate:
121  virtual void OnTestIterationStarting() OVERRIDE;
122  virtual std::string GetTestNameForFiltering(
123      const testing::TestCase* test_case,
124      const testing::TestInfo* test_info) OVERRIDE;
125  virtual bool ShouldRunTest(const testing::TestCase* test_case,
126                             const testing::TestInfo* test_info) OVERRIDE;
127  virtual size_t RunTests(base::TestLauncher* test_launcher,
128                          const std::vector<std::string>& test_names) OVERRIDE;
129  virtual size_t RetryTests(
130      base::TestLauncher* test_launcher,
131      const std::vector<std::string>& test_names) OVERRIDE;
132
133 private:
134  void DoRunTest(base::TestLauncher* test_launcher,
135                 const std::string& test_name);
136
137  // Launches test named |test_name| using parallel launcher,
138  // given result of PRE_ test |pre_test_result|.
139  void RunDependentTest(base::TestLauncher* test_launcher,
140                        const std::string test_name,
141                        const base::TestResult& pre_test_result);
142
143  // Callback to receive result of a test.
144  void GTestCallback(
145      base::TestLauncher* test_launcher,
146      const std::string& test_name,
147      int exit_code,
148      const base::TimeDelta& elapsed_time,
149      bool was_timeout,
150      const std::string& output);
151
152  content::TestLauncherDelegate* launcher_delegate_;
153
154  // Store dependent test name (map is indexed by full test name).
155  typedef std::map<std::string, std::string> DependentTestMap;
156  DependentTestMap dependent_test_map_;
157  DependentTestMap reverse_dependent_test_map_;
158
159  // Store unique data directory prefix for test names (without PRE_ prefixes).
160  // PRE_ tests and tests that depend on them must share the same
161  // data directory. Using test name as directory name leads to too long
162  // names (exceeding UNIX_PATH_MAX, which creates a problem with
163  // process_singleton_linux). Create a randomly-named temporary directory
164  // and keep track of the names so that PRE_ tests can still re-use them.
165  typedef std::map<std::string, base::FilePath> UserDataDirMap;
166  UserDataDirMap user_data_dir_map_;
167
168  // Store names of all seen tests to properly handle PRE_ tests.
169  std::set<std::string> all_test_names_;
170
171  // Temporary directory for user data directories.
172  base::ScopedTempDir temp_dir_;
173
174  DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
175};
176
177void WrapperTestLauncherDelegate::OnTestIterationStarting() {
178  dependent_test_map_.clear();
179  user_data_dir_map_.clear();
180}
181
182std::string WrapperTestLauncherDelegate::GetTestNameForFiltering(
183    const testing::TestCase* test_case,
184    const testing::TestInfo* test_info) {
185  return RemoveAnyPrePrefixes(
186      std::string(test_case->name()) + "." + test_info->name());
187}
188
189bool WrapperTestLauncherDelegate::ShouldRunTest(
190    const testing::TestCase* test_case,
191    const testing::TestInfo* test_info) {
192  all_test_names_.insert(
193      std::string(test_case->name()) + "." + test_info->name());
194
195  if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
196      !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
197    return false;
198  }
199
200  if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) {
201    // We will actually run PRE_ tests, but to ensure they run on the same shard
202    // as dependent tests, handle all these details internally.
203    return false;
204  }
205
206  return true;
207}
208
209std::string GetPreTestName(const std::string& full_name) {
210  size_t dot_pos = full_name.find('.');
211  CHECK_NE(dot_pos, std::string::npos);
212  std::string test_case_name = full_name.substr(0, dot_pos);
213  std::string test_name = full_name.substr(dot_pos + 1);
214  return test_case_name + "." + kPreTestPrefix + test_name;
215}
216
217size_t WrapperTestLauncherDelegate::RunTests(
218    base::TestLauncher* test_launcher,
219    const std::vector<std::string>& test_names) {
220  // Number of additional tests to run because of dependencies.
221  size_t additional_tests_to_run_count = 0;
222
223  // Compute dependencies of tests to be run.
224  for (size_t i = 0; i < test_names.size(); i++) {
225    std::string full_name(test_names[i]);
226    std::string pre_test_name(GetPreTestName(full_name));
227
228    while (ContainsKey(all_test_names_, pre_test_name)) {
229      additional_tests_to_run_count++;
230
231      DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
232      dependent_test_map_[pre_test_name] = full_name;
233
234      DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
235      reverse_dependent_test_map_[full_name] = pre_test_name;
236
237      full_name = pre_test_name;
238      pre_test_name = GetPreTestName(pre_test_name);
239    }
240  }
241
242  for (size_t i = 0; i < test_names.size(); i++) {
243    std::string full_name(test_names[i]);
244
245    // Make sure no PRE_ tests were requested explicitly.
246    DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
247
248    if (!ContainsKey(user_data_dir_map_, full_name)) {
249      base::FilePath temp_dir;
250      CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
251                                          FILE_PATH_LITERAL("d"), &temp_dir));
252      user_data_dir_map_[full_name] = temp_dir;
253    }
254
255    // If the test has any dependencies, get to the root and start with that.
256    while (ContainsKey(reverse_dependent_test_map_, full_name))
257      full_name = GetPreTestName(full_name);
258
259    DoRunTest(test_launcher, full_name);
260  }
261
262  return test_names.size() + additional_tests_to_run_count;
263}
264
265size_t WrapperTestLauncherDelegate::RetryTests(
266    base::TestLauncher* test_launcher,
267    const std::vector<std::string>& test_names) {
268  // List of tests we can kick off right now, depending on no other tests.
269  std::vector<std::string> tests_to_run_now;
270
271  // We retry at least the tests requested to retry.
272  std::set<std::string> test_names_set(test_names.begin(), test_names.end());
273
274  // In the face of PRE_ tests, we need to retry the entire chain of tests,
275  // from the very first one.
276  for (size_t i = 0; i < test_names.size(); i++) {
277    std::string test_name(test_names[i]);
278    while (ContainsKey(reverse_dependent_test_map_, test_name)) {
279      test_name = reverse_dependent_test_map_[test_name];
280      test_names_set.insert(test_name);
281    }
282  }
283
284  // Discard user data directories from any previous runs. Start with
285  // fresh state.
286  for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
287       i != user_data_dir_map_.end();
288       ++i) {
289    // Delete temporary directories now to avoid using too much space in /tmp.
290    if (!base::DeleteFile(i->second, true)) {
291      LOG(WARNING) << "Failed to delete " << i->second.value();
292    }
293  }
294  user_data_dir_map_.clear();
295
296  for (std::set<std::string>::const_iterator i = test_names_set.begin();
297       i != test_names_set.end();
298       ++i) {
299    std::string full_name(*i);
300
301    // Make sure PRE_ tests and tests that depend on them share the same
302    // data directory - based it on the test name without prefixes.
303    std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
304    if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
305      base::FilePath temp_dir;
306      CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
307                                          FILE_PATH_LITERAL("d"), &temp_dir));
308      user_data_dir_map_[test_name_no_pre] = temp_dir;
309    }
310
311    size_t dot_pos = full_name.find('.');
312    CHECK_NE(dot_pos, std::string::npos);
313    std::string test_case_name = full_name.substr(0, dot_pos);
314    std::string test_name = full_name.substr(dot_pos + 1);
315    std::string pre_test_name(
316        test_case_name + "." + kPreTestPrefix + test_name);
317    if (!ContainsKey(test_names_set, pre_test_name))
318      tests_to_run_now.push_back(full_name);
319  }
320
321  for (size_t i = 0; i < tests_to_run_now.size(); i++)
322    DoRunTest(test_launcher, tests_to_run_now[i]);
323
324  return test_names_set.size();
325}
326
327void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
328                                            const std::string& test_name) {
329  std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
330
331  CommandLine cmd_line(*CommandLine::ForCurrentProcess());
332  CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
333            &cmd_line, user_data_dir_map_[test_name_no_pre]));
334
335  CommandLine new_cmd_line(cmd_line.GetProgram());
336  CommandLine::SwitchMap switches = cmd_line.GetSwitches();
337
338  // Strip out gtest_output flag because otherwise we would overwrite results
339  // of the other tests.
340  switches.erase(base::kGTestOutputFlag);
341
342  for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
343       iter != switches.end(); ++iter) {
344    new_cmd_line.AppendSwitchNative(iter->first, iter->second);
345  }
346
347  // Always enable disabled tests.  This method is not called with disabled
348  // tests unless this flag was specified to the browser test executable.
349  new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
350  new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
351  new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
352
353  char* browser_wrapper = getenv("BROWSER_WRAPPER");
354
355  test_launcher->LaunchChildGTestProcess(
356      new_cmd_line,
357      browser_wrapper ? browser_wrapper : std::string(),
358      TestTimeouts::action_max_timeout(),
359      base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
360                 base::Unretained(this),
361                 test_launcher,
362                 test_name));
363}
364
365void WrapperTestLauncherDelegate::RunDependentTest(
366    base::TestLauncher* test_launcher,
367    const std::string test_name,
368    const base::TestResult& pre_test_result) {
369  if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
370    // Only run the dependent test if PRE_ test succeeded.
371    DoRunTest(test_launcher, test_name);
372  } else {
373    // Otherwise skip the test.
374    base::TestResult test_result;
375    test_result.full_name = test_name;
376    test_result.status = base::TestResult::TEST_SKIPPED;
377    test_launcher->OnTestFinished(test_result);
378
379    if (ContainsKey(dependent_test_map_, test_name)) {
380      RunDependentTest(test_launcher,
381                       dependent_test_map_[test_name],
382                       test_result);
383    }
384  }
385}
386
387void WrapperTestLauncherDelegate::GTestCallback(
388    base::TestLauncher* test_launcher,
389    const std::string& test_name,
390    int exit_code,
391    const base::TimeDelta& elapsed_time,
392    bool was_timeout,
393    const std::string& output) {
394  base::TestResult result;
395  result.full_name = test_name;
396
397  // TODO(phajdan.jr): Recognize crashes.
398  if (exit_code == 0)
399    result.status = base::TestResult::TEST_SUCCESS;
400  else if (was_timeout)
401    result.status = base::TestResult::TEST_TIMEOUT;
402  else
403    result.status = base::TestResult::TEST_FAILURE;
404
405  result.elapsed_time = elapsed_time;
406
407  result.output_snippet = GetTestOutputSnippet(result, output);
408
409  if (ContainsKey(dependent_test_map_, test_name)) {
410    RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
411  } else {
412    // No other tests depend on this, we can delete the temporary directory now.
413    // Do so to avoid too many temporary files using lots of disk space.
414    std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
415    if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
416      if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
417        LOG(WARNING) << "Failed to delete "
418                     << user_data_dir_map_[test_name_no_pre].value();
419      }
420      user_data_dir_map_.erase(test_name_no_pre);
421    }
422  }
423
424  test_launcher->OnTestFinished(result);
425}
426
427}  // namespace
428
429const char kHelpFlag[]   = "help";
430
431const char kLaunchAsBrowser[] = "as-browser";
432
433// See kManualTestPrefix above.
434const char kRunManualTestsFlag[] = "run-manual";
435
436const char kSingleProcessTestsFlag[]   = "single_process";
437
438
439TestLauncherDelegate::~TestLauncherDelegate() {
440}
441
442bool ShouldRunContentMain() {
443#if defined(OS_WIN) || defined(OS_LINUX)
444  CommandLine* command_line = CommandLine::ForCurrentProcess();
445  return command_line->HasSwitch(switches::kProcessType) ||
446         command_line->HasSwitch(kLaunchAsBrowser);
447#else
448  return false;
449#endif  // defined(OS_WIN) || defined(OS_LINUX)
450}
451
452int RunContentMain(int argc, char** argv,
453                   TestLauncherDelegate* launcher_delegate) {
454#if defined(OS_WIN)
455  sandbox::SandboxInterfaceInfo sandbox_info = {0};
456  InitializeSandboxInfo(&sandbox_info);
457  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
458      launcher_delegate->CreateContentMainDelegate());
459  return ContentMain(GetModuleHandle(NULL),
460                     &sandbox_info,
461                     chrome_main_delegate.get());
462#elif defined(OS_LINUX)
463  scoped_ptr<ContentMainDelegate> chrome_main_delegate(
464      launcher_delegate->CreateContentMainDelegate());
465  return ContentMain(argc, const_cast<const char**>(argv),
466                     chrome_main_delegate.get());
467#endif  // defined(OS_WIN)
468  NOTREACHED();
469  return 0;
470}
471
472int LaunchTests(TestLauncherDelegate* launcher_delegate,
473                int default_jobs,
474                int argc,
475                char** argv) {
476  DCHECK(!g_launcher_delegate);
477  g_launcher_delegate = launcher_delegate;
478
479  CommandLine::Init(argc, argv);
480  const CommandLine* command_line = CommandLine::ForCurrentProcess();
481
482  if (command_line->HasSwitch(kHelpFlag)) {
483    PrintUsage();
484    return 0;
485  }
486
487  if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
488      (command_line->HasSwitch(switches::kSingleProcess) &&
489       command_line->HasSwitch(base::kGTestFilterFlag)) ||
490      command_line->HasSwitch(base::kGTestListTestsFlag) ||
491      command_line->HasSwitch(base::kGTestHelpFlag)) {
492#if defined(OS_WIN)
493    if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
494      sandbox::SandboxInterfaceInfo sandbox_info;
495      InitializeSandboxInfo(&sandbox_info);
496      InitializeSandbox(&sandbox_info);
497    }
498#endif
499    return launcher_delegate->RunTestSuite(argc, argv);
500  }
501
502  if (ShouldRunContentMain())
503    return RunContentMain(argc, argv, launcher_delegate);
504
505  base::AtExitManager at_exit;
506  testing::InitGoogleTest(&argc, argv);
507  TestTimeouts::Initialize();
508
509  fprintf(stdout,
510      "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
511      "For debugging a test inside a debugger, use the\n"
512      "--gtest_filter=<your_test_name> flag along with either\n"
513      "--single_process (to run the test in one launcher/browser process) or\n"
514      "--single-process (to do the above, and also run Chrome in single-"
515          "process mode).\n");
516
517  base::MessageLoopForIO message_loop;
518
519  WrapperTestLauncherDelegate delegate(launcher_delegate);
520  base::TestLauncher launcher(&delegate, default_jobs);
521  bool success = launcher.Run(argc, argv);
522  return (success ? 0 : 1);
523}
524
525TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
526  return g_launcher_delegate;
527}
528
529}  // namespace content
530