tab_capture_performancetest.cc revision f2477e01787aa58f445919b809d89e252beef54f 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/basictypes.h"
6#include "base/command_line.h"
7#if defined(OS_MACOSX)
8#include "base/mac/mac_util.h"
9#endif
10#include "base/strings/stringprintf.h"
11#include "base/test/trace_event_analyzer.h"
12#include "base/win/windows_version.h"
13#include "chrome/browser/extensions/extension_apitest.h"
14#include "chrome/browser/extensions/extension_service.h"
15#include "chrome/browser/extensions/extension_test_message_listener.h"
16#include "chrome/browser/extensions/tab_helper.h"
17#include "chrome/browser/profiles/profile.h"
18#include "chrome/browser/ui/fullscreen/fullscreen_controller.h"
19#include "chrome/common/chrome_switches.h"
20#include "chrome/common/chrome_version_info.h"
21#include "chrome/common/extensions/features/base_feature_provider.h"
22#include "chrome/common/extensions/features/complex_feature.h"
23#include "chrome/common/extensions/features/simple_feature.h"
24#include "chrome/test/base/test_launcher_utils.h"
25#include "chrome/test/base/test_switches.h"
26#include "chrome/test/base/tracing.h"
27#include "content/public/browser/render_process_host.h"
28#include "content/public/browser/render_view_host.h"
29#include "content/public/common/content_switches.h"
30#include "extensions/common/feature_switch.h"
31#include "extensions/common/features/feature.h"
32#include "testing/gtest/include/gtest/gtest.h"
33#include "testing/perf/perf_test.h"
34#include "ui/compositor/compositor_switches.h"
35#include "ui/gl/gl_switches.h"
36
37namespace {
38
39const char kExtensionId[] = "ddchlicdkolnonkihahngkmmmjnjlkkf";
40
41enum TestFlags {
42  kUseGpu              = 1 << 0, // Only execute test if --enable-gpu was given
43                                 // on the command line.  This is required for
44                                 // tests that run on GPU.
45  kForceGpuComposited  = 1 << 1, // Force the test to use the compositor.
46  kDisableVsync        = 1 << 2, // Do not limit framerate to vertical refresh.
47                                 // when on GPU, nor to 60hz when not on GPU.
48  kTestThroughWebRTC   = 1 << 3, // Send captured frames through webrtc
49  kSmallWindow         = 1 << 4, // 1 = 800x600, 0 = 2000x1000
50
51  kScaleQualityMask    = 3 << 5, // two bits select which scaling quality
52  kScaleQualityDefault = 0 << 5, // to use on aura.
53  kScaleQualityFast    = 1 << 5,
54  kScaleQualityGood    = 2 << 5,
55  kScaleQualityBest    = 3 << 5,
56};
57
58class TabCapturePerformanceTest
59    : public ExtensionApiTest,
60      public testing::WithParamInterface<int> {
61 public:
62  TabCapturePerformanceTest() {}
63
64  bool HasFlag(TestFlags flag) const {
65    return (GetParam() & flag) == flag;
66  }
67
68  bool IsGpuAvailable() const {
69    return CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu");
70  }
71
72  std::string ScalingMethod() const {
73    switch (GetParam() & kScaleQualityMask) {
74      case kScaleQualityFast:
75        return "fast";
76      case kScaleQualityGood:
77        return "good";
78      case kScaleQualityBest:
79        return "best";
80      default:
81        return "";
82    }
83  }
84
85  std::string GetSuffixForTestFlags() {
86    std::string suffix;
87    if (HasFlag(kForceGpuComposited))
88      suffix += "_comp";
89    if (HasFlag(kUseGpu))
90      suffix += "_gpu";
91    if (HasFlag(kDisableVsync))
92      suffix += "_novsync";
93    if (HasFlag(kTestThroughWebRTC))
94      suffix += "_webrtc";
95    if (!ScalingMethod().empty())
96      suffix += "_scale" + ScalingMethod();
97    if (HasFlag(kSmallWindow))
98      suffix += "_small";
99    return suffix;
100  }
101
102  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
103    if (!ScalingMethod().empty()) {
104      command_line->AppendSwitchASCII(switches::kTabCaptureUpscaleQuality,
105                                      ScalingMethod());
106      command_line->AppendSwitchASCII(switches::kTabCaptureDownscaleQuality,
107                                      ScalingMethod());
108    }
109
110    // UI tests boot up render views starting from about:blank. This causes
111    // the renderer to start up thinking it cannot use the GPU. To work
112    // around that, and allow the frame rate test to use the GPU, we must
113    // pass kAllowWebUICompositing.
114    command_line->AppendSwitch(switches::kAllowWebUICompositing);
115
116    // Some of the tests may launch http requests through JSON or AJAX
117    // which causes a security error (cross domain request) when the page
118    // is loaded from the local file system ( file:// ). The following switch
119    // fixes that error.
120    command_line->AppendSwitch(switches::kAllowFileAccessFromFiles);
121
122    if (HasFlag(kSmallWindow)) {
123      command_line->AppendSwitchASCII(switches::kWindowSize, "800,600");
124    } else {
125      command_line->AppendSwitchASCII(switches::kWindowSize, "2000,1500");
126    }
127
128    UseRealGLContexts();
129
130    if (!HasFlag(kUseGpu)) {
131      command_line->AppendSwitch(switches::kDisableGpu);
132    } else {
133      command_line->AppendSwitch(switches::kForceCompositingMode);
134    }
135
136    if (HasFlag(kDisableVsync))
137      command_line->AppendSwitch(switches::kDisableGpuVsync);
138
139    command_line->AppendSwitchASCII(switches::kWhitelistedExtensionID,
140                                    kExtensionId);
141    ExtensionApiTest::SetUpCommandLine(command_line);
142  }
143
144  bool PrintResults(trace_analyzer::TraceAnalyzer *analyzer,
145                    const std::string& test_name,
146                    const std::string& event_name,
147                    const std::string& unit) {
148    trace_analyzer::TraceEventVector events;
149    trace_analyzer::Query query =
150        trace_analyzer::Query::EventNameIs(event_name) &&
151        (trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_BEGIN) ||
152         trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_ASYNC_BEGIN) ||
153         trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_FLOW_BEGIN) ||
154         trace_analyzer::Query::EventPhaseIs(TRACE_EVENT_PHASE_INSTANT));
155    analyzer->FindEvents(query, &events);
156    if (events.size() < 20) {
157      LOG(INFO) << "Not enough events of type " << event_name << " found.";
158      return false;
159    }
160
161    // Ignore some events for startup/setup/caching.
162    trace_analyzer::TraceEventVector rate_events(events.begin() + 3,
163                                                 events.end() - 3);
164    trace_analyzer::RateStats stats;
165    if (!GetRateStats(rate_events, &stats, NULL)) {
166      LOG(INFO) << "GetRateStats failed";
167      return false;
168    }
169    double mean_ms = stats.mean_us / 1000.0;
170    double std_dev_ms = stats.standard_deviation_us / 1000.0;
171    std::string mean_and_error = base::StringPrintf("%f,%f", mean_ms,
172                                                    std_dev_ms);
173    perf_test::PrintResultMeanAndError(test_name,
174                                       GetSuffixForTestFlags(),
175                                       event_name,
176                                       mean_and_error,
177                                       unit,
178                                       true);
179    return true;
180  }
181
182  void RunTest(const std::string& test_name) {
183    if (HasFlag(kUseGpu) && !IsGpuAvailable()) {
184      LOG(WARNING) <<
185          "Test skipped: requires gpu. Pass --enable-gpu on the command "
186          "line if use of GPU is desired.";
187      return;
188    }
189
190    std::string json_events;
191    ASSERT_TRUE(tracing::BeginTracing("test_fps,mirroring"));
192    std::string page = "performance.html";
193    page += HasFlag(kTestThroughWebRTC) ? "?WebRTC=1" : "?WebRTC=0";
194    // Ideally we'd like to run a higher capture rate when vsync is disabled,
195    // but libjingle currently doesn't allow that.
196    // page += HasFlag(kDisableVsync) ? "&fps=300" : "&fps=30";
197    page += "&fps=30";
198    ASSERT_TRUE(RunExtensionSubtest("tab_capture/experimental", page))
199        << message_;
200    ASSERT_TRUE(tracing::EndTracing(&json_events));
201    scoped_ptr<trace_analyzer::TraceAnalyzer> analyzer;
202    analyzer.reset(trace_analyzer::TraceAnalyzer::Create(json_events));
203
204    // Only one of these PrintResults should actually print something.
205    // The printed result will be the average time between frames in the
206    // browser window.
207    bool sw_frames = PrintResults(analyzer.get(),
208                                  test_name,
209                                  "TestFrameTickSW",
210                                  "frame_time");
211    bool gpu_frames = PrintResults(analyzer.get(),
212                                   test_name,
213                                   "TestFrameTickGPU",
214                                   "frame_time");
215    EXPECT_TRUE(sw_frames || gpu_frames);
216    EXPECT_NE(sw_frames, gpu_frames);
217
218    // This prints out the average time between capture events.
219    // As the capture frame rate is capped at 30fps, this score
220    // cannot get any better than 33.33 ms.
221    EXPECT_TRUE(PrintResults(analyzer.get(),
222                             test_name,
223                             "Capture",
224                             "capture_time"));
225  }
226};
227
228}  // namespace
229
230IN_PROC_BROWSER_TEST_P(TabCapturePerformanceTest, Performance) {
231  RunTest("TabCapturePerformance");
232}
233
234// Note: First argument is optional and intentionally left blank.
235// (it's a prefix for the generated test cases)
236INSTANTIATE_TEST_CASE_P(
237    ,
238    TabCapturePerformanceTest,
239    testing::Values(
240        0,
241        kUseGpu | kForceGpuComposited,
242        kDisableVsync,
243        kDisableVsync | kUseGpu | kForceGpuComposited,
244        kTestThroughWebRTC,
245        kTestThroughWebRTC | kUseGpu | kForceGpuComposited,
246        kTestThroughWebRTC | kDisableVsync,
247        kTestThroughWebRTC | kDisableVsync | kUseGpu | kForceGpuComposited));
248
249#if defined(USE_AURA)
250// TODO(hubbe):
251// These are temporary tests for the purpose of determining what the
252// appropriate scaling quality is. Once that has been determined,
253// these tests will be removed.
254
255const int kScalingTestBase =
256    kTestThroughWebRTC | kDisableVsync | kUseGpu | kForceGpuComposited;
257
258INSTANTIATE_TEST_CASE_P(
259    ScalingTests,
260    TabCapturePerformanceTest,
261    testing::Values(
262        kScalingTestBase | kScaleQualityFast,
263        kScalingTestBase | kScaleQualityGood,
264        kScalingTestBase | kScaleQualityBest,
265        kScalingTestBase | kScaleQualityFast | kSmallWindow,
266        kScalingTestBase | kScaleQualityGood | kSmallWindow,
267        kScalingTestBase | kScaleQualityBest | kSmallWindow));
268
269#endif  // USE_AURA
270