1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/environment.h"
6#include "base/file_util.h"
7#include "base/files/file_enumerator.h"
8#include "base/path_service.h"
9#include "base/strings/string_number_conversions.h"
10#include "base/strings/string_split.h"
11#include "base/strings/string_util.h"
12#include "base/strings/stringprintf.h"
13#include "base/strings/utf_string_conversions.h"
14#include "base/sys_info.h"
15#include "base/test/test_file_util.h"
16#include "base/test/test_timeouts.h"
17#include "base/time/time.h"
18#include "chrome/common/chrome_constants.h"
19#include "chrome/common/chrome_paths.h"
20#include "chrome/common/chrome_switches.h"
21#include "chrome/common/env_vars.h"
22#include "chrome/test/automation/automation_proxy.h"
23#include "chrome/test/base/test_switches.h"
24#include "chrome/test/base/testing_profile.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "chrome/test/perf/perf_test.h"
27#include "chrome/test/perf/perf_ui_test_suite.h"
28#include "chrome/test/ui/ui_perf_test.h"
29#include "content/public/common/content_switches.h"
30#include "net/base/net_util.h"
31
32using base::TimeDelta;
33using base::TimeTicks;
34
35namespace {
36
37class StartupTest : public UIPerfTest {
38 public:
39  StartupTest() {
40    show_window_ = true;
41  }
42  virtual void SetUp() {
43    collect_profiling_stats_ = false;
44  }
45  virtual void TearDown() {}
46
47  enum TestColdness {
48    WARM,
49    COLD
50  };
51
52  enum TestImportance {
53    NOT_IMPORTANT,
54    IMPORTANT
55  };
56
57  // Load a file on startup rather than about:blank.  This tests a longer
58  // startup path, including resource loading.
59  void SetUpWithFileURL() {
60    const base::FilePath file_url = ui_test_utils::GetTestFilePath(
61        base::FilePath(base::FilePath::kCurrentDirectory),
62        base::FilePath(FILE_PATH_LITERAL("simple.html")));
63    ASSERT_TRUE(base::PathExists(file_url));
64    launch_arguments_.AppendArgPath(file_url);
65  }
66
67  // Setup the command line arguments to capture profiling data for tasks.
68  void SetUpWithProfiling() {
69    profiling_file_ = ui_test_utils::GetTestFilePath(
70        base::FilePath(base::FilePath::kCurrentDirectory),
71        base::FilePath(FILE_PATH_LITERAL("task_profile.json")));
72    launch_arguments_.AppendSwitchPath(switches::kProfilingOutputFile,
73                                       profiling_file_);
74    collect_profiling_stats_ = true;
75  }
76
77  // Load a complex html file on startup represented by |which_tab|.
78  void SetUpWithComplexFileURL(unsigned int which_tab) {
79    const char* const kTestPageCyclerDomains[] = {
80        "www.google.com", "www.nytimes.com",
81        "www.yahoo.com", "espn.go.com", "www.amazon.com"
82    };
83    unsigned int which_el = which_tab % arraysize(kTestPageCyclerDomains);
84    const char* this_domain = kTestPageCyclerDomains[which_el];
85
86    base::FilePath page_cycler_path;
87    PathService::Get(base::DIR_SOURCE_ROOT, &page_cycler_path);
88    page_cycler_path = page_cycler_path.AppendASCII("data")
89        .AppendASCII("page_cycler").AppendASCII("moz")
90        .AppendASCII(this_domain).AppendASCII("index.html");
91    GURL file_url = net::FilePathToFileURL(page_cycler_path).Resolve("?skip");
92    launch_arguments_.AppendArg(file_url.spec());
93  }
94
95  // Use the given profile in the test data extensions/profiles dir.  This tests
96  // startup with extensions installed.
97  void SetUpWithExtensionsProfile(const char* profile) {
98    base::FilePath data_dir;
99    PathService::Get(chrome::DIR_TEST_DATA, &data_dir);
100    data_dir = data_dir.AppendASCII("extensions").AppendASCII("profiles").
101        AppendASCII(profile);
102    set_template_user_data(data_dir);
103  }
104
105  // Rewrite the preferences file to point to the proper image directory.
106  virtual void SetUpProfile() OVERRIDE {
107    UIPerfTest::SetUpProfile();
108    if (profile_type_ != PerfUITestSuite::COMPLEX_THEME)
109      return;
110
111    const base::FilePath pref_template_path(user_data_dir().
112        AppendASCII("Default").
113        AppendASCII("PreferencesTemplate"));
114    const base::FilePath pref_path(user_data_dir().
115        AppendASCII(TestingProfile::kTestUserProfileDir).
116        AppendASCII("Preferences"));
117
118    // Read in preferences template.
119    std::string pref_string;
120    EXPECT_TRUE(file_util::ReadFileToString(pref_template_path, &pref_string));
121    string16 format_string = ASCIIToUTF16(pref_string);
122
123    // Make sure temp directory has the proper format for writing to prefs file.
124#if defined(OS_POSIX)
125    std::wstring user_data_dir_w(ASCIIToWide(user_data_dir().value()));
126#elif defined(OS_WIN)
127    std::wstring user_data_dir_w(user_data_dir().value());
128    // In Windows, the FilePath will write '\' for the path separators; change
129    // these to a separator that won't trigger escapes.
130    std::replace(user_data_dir_w.begin(),
131                 user_data_dir_w.end(), '\\', '/');
132#endif
133
134    // Rewrite prefs file.
135    std::vector<string16> subst;
136    subst.push_back(WideToUTF16(user_data_dir_w));
137    const std::string prefs_string =
138        UTF16ToASCII(ReplaceStringPlaceholders(format_string, subst, NULL));
139    EXPECT_TRUE(file_util::WriteFile(pref_path, prefs_string.c_str(),
140                                     prefs_string.size()));
141    file_util::EvictFileFromSystemCache(pref_path);
142  }
143
144  // Runs a test which loads |tab_count| tabs on startup, either as command line
145  // arguments or, if |restore_session| is true, by using session restore.
146  // |nth_timed_tab|, if non-zero, will measure time to load the first n+1 tabs.
147  void RunPerfTestWithManyTabs(const char* graph, const char* trace,
148                               int tab_count, int nth_timed_tab,
149                               bool restore_session);
150
151  void RunStartupTest(const char* graph, const char* trace,
152                      TestColdness test_cold, TestImportance test_importance,
153                      PerfUITestSuite::ProfileType profile_type,
154                      int num_tabs, int nth_timed_tab) {
155    bool important = (test_importance == IMPORTANT);
156    profile_type_ = profile_type;
157
158    // Sets the profile data for the run.  For now, this is only used for
159    // the non-default themes test.
160    if (profile_type != PerfUITestSuite::DEFAULT_THEME) {
161      set_template_user_data(
162          PerfUITestSuite::GetPathForProfileType(profile_type));
163    }
164
165#if defined(NDEBUG)
166    const int kNumCyclesMax = 20;
167#else
168    // Debug builds are too slow and we can't run that many cycles in a
169    // reasonable amount of time.
170    const int kNumCyclesMax = 10;
171#endif
172    int numCycles = kNumCyclesMax;
173    scoped_ptr<base::Environment> env(base::Environment::Create());
174    std::string numCyclesEnv;
175    if (env->GetVar(env_vars::kStartupTestsNumCycles, &numCyclesEnv) &&
176        base::StringToInt(numCyclesEnv, &numCycles)) {
177      if (numCycles <= kNumCyclesMax) {
178        VLOG(1) << env_vars::kStartupTestsNumCycles
179                << " set in environment, so setting numCycles to " << numCycles;
180      } else {
181        VLOG(1) << env_vars::kStartupTestsNumCycles
182                << " is higher than the max, setting numCycles to "
183                << kNumCyclesMax;
184        numCycles = kNumCyclesMax;
185      }
186    }
187
188    struct TimingInfo {
189      TimeDelta end_to_end;
190      float first_start_ms;
191      float last_stop_ms;
192      float first_stop_ms;
193      float nth_tab_stop_ms;
194    };
195    TimingInfo timings[kNumCyclesMax];
196
197    CommandLine launch_arguments_without_trace_file(launch_arguments_);
198    for (int i = 0; i < numCycles; ++i) {
199      if (test_cold == COLD) {
200        base::FilePath dir_app;
201        ASSERT_TRUE(PathService::Get(chrome::DIR_APP, &dir_app));
202
203        base::FilePath chrome_exe(dir_app.Append(GetExecutablePath()));
204        ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_exe));
205#if defined(OS_WIN)
206        // chrome.dll is windows specific.
207        base::FilePath chrome_dll(
208            dir_app.Append(FILE_PATH_LITERAL("chrome.dll")));
209        ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_dll));
210#endif
211      }
212      UITest::SetUp();
213      TimeTicks end_time = TimeTicks::Now();
214
215      if (num_tabs > 0) {
216        float min_start;
217        float max_stop;
218        std::vector<float> times;
219        scoped_refptr<BrowserProxy> browser_proxy(
220            automation()->GetBrowserWindow(0));
221        ASSERT_TRUE(browser_proxy.get());
222
223        if (browser_proxy->GetInitialLoadTimes(
224              TestTimeouts::action_max_timeout(),
225              &min_start,
226              &max_stop,
227              &times) &&
228            !times.empty()) {
229          ASSERT_LT(nth_timed_tab, num_tabs);
230          ASSERT_EQ(times.size(), static_cast<size_t>(num_tabs));
231          timings[i].first_start_ms = min_start;
232          timings[i].last_stop_ms = max_stop;
233          timings[i].first_stop_ms = times[0];
234          timings[i].nth_tab_stop_ms = times[nth_timed_tab];
235        } else {
236          // Browser might not support initial load times.
237          // Only use end-to-end time for this test.
238          num_tabs = 0;
239        }
240      }
241      timings[i].end_to_end = end_time - browser_launch_time();
242      UITest::TearDown();
243
244      if (i == 0) {
245        // Re-use the profile data after first run so that the noise from
246        // creating databases doesn't impact all the runs.
247        clear_profile_ = false;
248        // Clear template_user_data_ so we don't try to copy it over each time
249        // through.
250        set_template_user_data(base::FilePath());
251      }
252    }
253
254    std::string times;
255    for (int i = 0; i < numCycles; ++i) {
256      base::StringAppendF(&times,
257                          "%.2f,",
258                          timings[i].end_to_end.InMillisecondsF());
259    }
260    perf_test::PrintResultList(
261        graph, std::string(), trace, times, "ms", important);
262
263    if (num_tabs > 0) {
264      std::string name_base = trace;
265      std::string name;
266
267      times.clear();
268      name = name_base + "-start";
269      for (int i = 0; i < numCycles; ++i)
270        base::StringAppendF(&times, "%.2f,", timings[i].first_start_ms);
271      perf_test::PrintResultList(
272          graph, std::string(), name.c_str(), times, "ms", important);
273
274      times.clear();
275      name = name_base + "-first";
276      for (int i = 0; i < numCycles; ++i)
277        base::StringAppendF(&times, "%.2f,", timings[i].first_stop_ms);
278      perf_test::PrintResultList(
279          graph, std::string(), name.c_str(), times, "ms", important);
280
281      if (nth_timed_tab > 0) {
282        // Display only the time necessary to load the first n tabs.
283        times.clear();
284        name = name_base + "-" + base::IntToString(nth_timed_tab);
285        for (int i = 0; i < numCycles; ++i)
286          base::StringAppendF(&times, "%.2f,", timings[i].nth_tab_stop_ms);
287        perf_test::PrintResultList(
288            graph, std::string(), name.c_str(), times, "ms", important);
289      }
290
291      if (num_tabs > 1) {
292        // Display the time necessary to load all of the tabs.
293        times.clear();
294        name = name_base + "-all";
295        for (int i = 0; i < numCycles; ++i)
296          base::StringAppendF(&times, "%.2f,", timings[i].last_stop_ms);
297        perf_test::PrintResultList(
298            graph, std::string(), name.c_str(), times, "ms", important);
299      }
300    }
301  }
302
303  base::FilePath profiling_file_;
304  bool collect_profiling_stats_;
305  std::string trace_file_prefix_;
306
307  // Are we using a profile with a complex theme?
308  PerfUITestSuite::ProfileType profile_type_;
309};
310
311TEST_F(StartupTest, PerfWarm) {
312  RunStartupTest("warm", "t", WARM, IMPORTANT,
313                 PerfUITestSuite::DEFAULT_THEME, 0, 0);
314}
315
316TEST_F(StartupTest, PerfReferenceWarm) {
317  UseReferenceBuild();
318  RunStartupTest("warm", "t_ref", WARM, IMPORTANT,
319                 PerfUITestSuite::DEFAULT_THEME, 0, 0);
320}
321
322// TODO(mpcomplete): Should we have reference timings for all these?
323
324// dominich: Disabling as per http://crbug.com/100900.
325#if defined(OS_WIN)
326#define MAYBE_PerfCold DISABLED_PerfCold
327#else
328#define MAYBE_PerfCold PerfCold
329#endif
330
331TEST_F(StartupTest, MAYBE_PerfCold) {
332  RunStartupTest("cold", "t", COLD, NOT_IMPORTANT,
333                 PerfUITestSuite::DEFAULT_THEME, 0, 0);
334}
335
336void StartupTest::RunPerfTestWithManyTabs(const char* graph, const char* trace,
337                                          int tab_count, int nth_timed_tab,
338                                          bool restore_session) {
339  // Initialize session with |tab_count| tabs.
340  for (int i = 0; i < tab_count; ++i)
341    SetUpWithComplexFileURL(i);
342
343  if (restore_session) {
344    // Start the browser with these urls so we can save the session and exit.
345    UITest::SetUp();
346    // Set flags to ensure profile is saved and can be restored.
347#if defined(OS_CHROMEOS) || defined(OS_MACOSX)
348    set_shutdown_type(ProxyLauncher::USER_QUIT);
349#endif
350    clear_profile_ = false;
351    // Quit and set flags to restore session.
352    UITest::TearDown();
353
354    // Clear all arguments for session restore, or the number of open tabs
355    // will grow with each restore.
356    CommandLine new_launch_arguments = CommandLine(CommandLine::NO_PROGRAM);
357    // Keep the branding switch if using a reference build.
358    if (launch_arguments_.HasSwitch(switches::kEnableChromiumBranding)) {
359      new_launch_arguments.AppendSwitch(switches::kEnableChromiumBranding);
360    }
361    // The session will be restored once per cycle for numCycles test cycles,
362    // and each time, UITest::SetUp will wait for |tab_count| tabs to
363    // finish loading.
364    new_launch_arguments.AppendSwitchASCII(switches::kRestoreLastSession,
365                                        base::IntToString(tab_count));
366    launch_arguments_ = new_launch_arguments;
367  }
368  RunStartupTest(graph, trace, WARM, NOT_IMPORTANT,
369                 PerfUITestSuite::DEFAULT_THEME, tab_count, nth_timed_tab);
370}
371
372// http://crbug.com/101591
373#if defined(OS_WIN) && !defined(NDEBUG)
374#define MAYBE_PerfFewTabs DISABLED_PerfFewTabs
375#else
376#define MAYBE_PerfFewTabs PerfFewTabs
377#endif
378
379TEST_F(StartupTest, MAYBE_PerfFewTabs) {
380  RunPerfTestWithManyTabs("few_tabs", "cmdline", 5, 2, false);
381}
382
383TEST_F(StartupTest, PerfFewTabsReference) {
384  UseReferenceBuild();
385  RunPerfTestWithManyTabs("few_tabs", "cmdline-ref", 5, 2, false);
386}
387
388TEST_F(StartupTest, PerfRestoreFewTabs) {
389  RunPerfTestWithManyTabs("few_tabs", "restore", 5, 2, true);
390}
391
392TEST_F(StartupTest, PerfRestoreFewTabsReference) {
393  UseReferenceBuild();
394  RunPerfTestWithManyTabs("few_tabs", "restore-ref", 5, 2, true);
395}
396
397#if defined(OS_MACOSX)
398// http://crbug.com/46609
399#define MAYBE_PerfSeveralTabsReference DISABLED_PerfSeveralTabsReference
400#define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
401// http://crbug.com/52858
402#define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
403#define MAYBE_PerfExtensionContentScript50 DISABLED_PerfExtensionContentScript50
404#elif defined(OS_WIN)
405// http://crbug.com/46609
406#if !defined(NDEBUG)
407// This test is disabled on Windows Debug. See bug http://crbug.com/132279
408#define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
409#else
410#define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
411#endif  // !defined(NDEBUG)
412// http://crbug.com/102584
413#define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
414#define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
415#define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
416#else
417#define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
418#define MAYBE_PerfSeveralTabs PerfSeveralTabs
419#define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
420#define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
421#endif
422
423// http://crbug.com/99604
424#if defined(OS_WIN) && !defined(NDEBUG)
425#define MAYBE_PerfComplexTheme DISABLED_PerfComplexTheme
426#else
427#define MAYBE_PerfComplexTheme PerfComplexTheme
428#endif
429
430TEST_F(StartupTest, MAYBE_PerfSeveralTabs) {
431  RunPerfTestWithManyTabs("several_tabs", "cmdline", 10, 4, false);
432}
433
434TEST_F(StartupTest, MAYBE_PerfSeveralTabsReference) {
435  UseReferenceBuild();
436  RunPerfTestWithManyTabs("several_tabs", "cmdline-ref", 10, 4, false);
437}
438
439TEST_F(StartupTest, MAYBE_PerfRestoreSeveralTabs) {
440  RunPerfTestWithManyTabs("several_tabs", "restore", 10, 4, true);
441}
442
443TEST_F(StartupTest, PerfRestoreSeveralTabsReference) {
444  UseReferenceBuild();
445  RunPerfTestWithManyTabs("several_tabs", "restore-ref", 10, 4, true);
446}
447
448TEST_F(StartupTest, PerfExtensionEmpty) {
449  SetUpWithFileURL();
450  SetUpWithExtensionsProfile("empty");
451  RunStartupTest("warm", "extension_empty", WARM, NOT_IMPORTANT,
452                 PerfUITestSuite::DEFAULT_THEME, 1, 0);
453}
454
455TEST_F(StartupTest, PerfExtensionContentScript1) {
456  SetUpWithFileURL();
457  SetUpWithExtensionsProfile("content_scripts1");
458  RunStartupTest("warm", "extension_content_scripts1", WARM, NOT_IMPORTANT,
459                 PerfUITestSuite::DEFAULT_THEME, 1, 0);
460}
461
462TEST_F(StartupTest, MAYBE_PerfExtensionContentScript50) {
463  SetUpWithFileURL();
464  SetUpWithExtensionsProfile("content_scripts50");
465  RunStartupTest("warm", "extension_content_scripts50", WARM, NOT_IMPORTANT,
466                 PerfUITestSuite::DEFAULT_THEME, 1, 0);
467}
468
469TEST_F(StartupTest, MAYBE_PerfComplexTheme) {
470  RunStartupTest("warm", "t-theme", WARM, NOT_IMPORTANT,
471                 PerfUITestSuite::COMPLEX_THEME, 0, 0);
472}
473
474TEST_F(StartupTest, ProfilingScript1) {
475  SetUpWithFileURL();
476  SetUpWithProfiling();
477  RunStartupTest("warm", "profiling_scripts1", WARM, NOT_IMPORTANT,
478                 PerfUITestSuite::DEFAULT_THEME, 1, 0);
479}
480
481}  // namespace
482