chrome_webrtc_video_quality_browsertest.cc revision d0247b1b59f9c528cb6df88b4f2b9afaf80d181e 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/environment.h"
6#include "base/file_util.h"
7#include "base/path_service.h"
8#include "base/process/launch.h"
9#include "base/strings/string_split.h"
10#include "base/strings/stringprintf.h"
11#include "base/test/test_timeouts.h"
12#include "base/time/time.h"
13#include "chrome/browser/chrome_notification_types.h"
14#include "chrome/browser/infobars/infobar.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "chrome/test/ui/ui_test.h"
27#include "content/public/browser/notification_service.h"
28#include "content/public/test/browser_test_utils.h"
29#include "net/test/embedded_test_server/embedded_test_server.h"
30#include "net/test/python_utils.h"
31#include "testing/perf/perf_test.h"
32
33static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
34#if defined(OS_WIN)
35    FILE_PATH_LITERAL("frame_analyzer.exe");
36#else
37    FILE_PATH_LITERAL("frame_analyzer");
38#endif
39
40static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
41#if defined(OS_WIN)
42    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
43#else
44    FILE_PATH_LITERAL("rgba_to_i420_converter");
45#endif
46
47static const char kHomeEnvName[] =
48#if defined(OS_WIN)
49    "HOMEPATH";
50#else
51    "HOME";
52#endif
53
54// The working dir should be in the user's home folder.
55static const base::FilePath::CharType kWorkingDirName[] =
56    FILE_PATH_LITERAL("webrtc_video_quality");
57static const base::FilePath::CharType kReferenceYuvFileName[] =
58    FILE_PATH_LITERAL("reference_video.yuv");
59static const base::FilePath::CharType kCapturedYuvFileName[] =
60    FILE_PATH_LITERAL("captured_video.yuv");
61static const base::FilePath::CharType kStatsFileName[] =
62    FILE_PATH_LITERAL("stats.txt");
63static const char kMainWebrtcTestHtmlPage[] =
64    "/webrtc/webrtc_jsep01_test.html";
65static const char kCapturingWebrtcHtmlPage[] =
66    "/webrtc/webrtc_video_quality_test.html";
67static const int kVgaWidth = 640;
68static const int kVgaHeight = 480;
69
70// If you change the port number, don't forget to modify video_extraction.js
71// too!
72static const char kPyWebSocketPortNumber[] = "12221";
73
74// Test the video quality of the WebRTC output.
75//
76// Prerequisites: This test case must run on a machine with a virtual webcam
77// that plays video from the reference file located in <the running user's home
78// folder>/kWorkingDirName/kReferenceYuvFileName.
79//
80// You must also compile the chromium_builder_webrtc target before you run this
81// test to get all the tools built.
82//
83// The external compare_videos.py script also depends on two external
84// executables which must be located in the PATH when running this test.
85// * zxing (see the CPP version at https://code.google.com/p/zxing)
86// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
87//
88// The test case will launch a custom binary (peerconnection_server) which will
89// allow two WebRTC clients to find each other.
90//
91// The test also runs several other custom binaries - rgba_to_i420 converter and
92// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
93// test also runs a stand alone Python implementation of a WebSocket server
94// (pywebsocket) and a barcode_decoder script.
95class WebrtcVideoQualityBrowserTest : public WebRtcTestBase {
96 public:
97  WebrtcVideoQualityBrowserTest()
98      : pywebsocket_server_(0),
99        environment_(base::Environment::Create()) {}
100
101  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
102    PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem();
103  }
104
105  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
106    // TODO(phoglund): check that user actually has the requisite devices and
107    // print a nice message if not; otherwise the test just times out which can
108    // be confusing.
109    // This test expects real device handling and requires a real webcam / audio
110    // device; it will not work with fake devices.
111    EXPECT_FALSE(
112        command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream))
113        << "You cannot run this test with fake devices.";
114
115    // The video playback will not work without a GPU, so force its use here.
116    command_line->AppendSwitch(switches::kUseGpuInTests);
117  }
118
119  bool HasAllRequiredResources() {
120    if (!base::PathExists(GetWorkingDir())) {
121      LOG(ERROR) << "Cannot find the working directory for the reference video "
122          "and the temporary files:" << GetWorkingDir().value();
123      return false;
124    }
125    base::FilePath reference_file =
126        GetWorkingDir().Append(kReferenceYuvFileName);
127    if (!base::PathExists(reference_file)) {
128      LOG(ERROR) << "Cannot find the reference file to be used for video "
129          << "quality comparison: " << reference_file.value();
130      return false;
131    }
132    return true;
133  }
134
135  bool StartPyWebSocketServer() {
136    base::FilePath path_pywebsocket_dir =
137        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
138    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
139        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
140    base::FilePath path_to_data_handler =
141        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));
142
143    if (!base::PathExists(pywebsocket_server)) {
144      LOG(ERROR) << "Missing pywebsocket server.";
145      return false;
146    }
147    if (!base::PathExists(path_to_data_handler)) {
148      LOG(ERROR) << "Missing data handler for pywebsocket server.";
149      return false;
150    }
151
152    AppendToPythonPath(path_pywebsocket_dir);
153
154    // Note: don't append switches to this command since it will mess up the
155    // -u in the python invocation!
156    CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
157    EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
158
159    pywebsocket_command.AppendArgPath(pywebsocket_server);
160    pywebsocket_command.AppendArg("-p");
161    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
162    pywebsocket_command.AppendArg("-d");
163    pywebsocket_command.AppendArgPath(path_to_data_handler);
164
165    LOG(INFO) << "Running " << pywebsocket_command.GetCommandLineString();
166    return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
167                               &pywebsocket_server_);
168  }
169
170  bool ShutdownPyWebSocketServer() {
171    return base::KillProcess(pywebsocket_server_, 0, false);
172  }
173
174  // Convenience method which executes the provided javascript in the context
175  // of the provided web contents and returns what it evaluated to.
176  std::string ExecuteJavascript(const std::string& javascript,
177                                content::WebContents* tab_contents) {
178    std::string result;
179    EXPECT_TRUE(content::ExecuteScriptAndExtractString(
180        tab_contents, javascript, &result));
181    return result;
182  }
183
184  // Ensures we didn't get any errors asynchronously (e.g. while no javascript
185  // call from this test was outstanding).
186  // TODO(phoglund): this becomes obsolete when we switch to communicating with
187  // the DOM message queue.
188  void AssertNoAsynchronousErrors(content::WebContents* tab_contents) {
189    EXPECT_EQ("ok-no-errors",
190              ExecuteJavascript("getAnyTestFailures()", tab_contents));
191  }
192
193  // The peer connection server lets our two tabs find each other and talk to
194  // each other (e.g. it is the application-specific "signaling solution").
195  void ConnectToPeerConnectionServer(const std::string peer_name,
196                                     content::WebContents* tab_contents) {
197    std::string javascript = base::StringPrintf(
198        "connect('http://localhost:8888', '%s');", peer_name.c_str());
199    EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents));
200  }
201
202  void EstablishCall(content::WebContents* from_tab,
203                     content::WebContents* to_tab) {
204    EXPECT_EQ("ok-peerconnection-created",
205              ExecuteJavascript("preparePeerConnection()", from_tab));
206    EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab));
207    EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab));
208
209    // Ensure the call gets up on both sides.
210    EXPECT_TRUE(PollingWaitUntil(
211        "getPeerConnectionReadyState()", "active", from_tab));
212    EXPECT_TRUE(PollingWaitUntil(
213        "getPeerConnectionReadyState()", "active", to_tab));
214  }
215
216  void HangUp(content::WebContents* from_tab) {
217    EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
218  }
219
220  void WaitUntilHangupVerified(content::WebContents* tab_contents) {
221    EXPECT_TRUE(PollingWaitUntil(
222        "getPeerConnectionReadyState()", "no-peer-connection", tab_contents));
223  }
224
225  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
226  // which should contain frames of size |width| x |height|.
227  //
228  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
229  // should be build prior to running this test. The resulting binary should
230  // live next to Chrome.
231  bool RunARGBtoI420Converter(int width,
232                              int height,
233                              const base::FilePath& captured_video_filename) {
234    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
235        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
236
237    if (!base::PathExists(path_to_converter)) {
238      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
239          << path_to_converter.value();
240      return false;
241    }
242
243    CommandLine converter_command(path_to_converter);
244    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
245    converter_command.AppendSwitchPath("--output_file",
246                                       captured_video_filename);
247    converter_command.AppendSwitchASCII("--width",
248                                        base::StringPrintf("%d", width));
249    converter_command.AppendSwitchASCII("--height",
250                                        base::StringPrintf("%d", height));
251
252    // We produce an output file that will later be used as an input to the
253    // barcode decoder and frame analyzer tools.
254    LOG(INFO) << "Running " << converter_command.GetCommandLineString();
255    std::string result;
256    bool ok = base::GetAppOutput(converter_command, &result);
257    LOG(INFO) << "Output was:\n\n" << result;
258    return ok;
259  }
260
261  // Compares the |captured_video_filename| with the |reference_video_filename|.
262  //
263  // The barcode decoder decodes the captured video containing barcodes overlaid
264  // into every frame of the video (produced by rgba_to_i420_converter). It
265  // produces a set of PNG images and a |stats_file| that maps each captured
266  // frame to a frame in the reference video. The frames should be of size
267  // |width| x |height|.
268  // All measurements calculated are printed as perf parsable numbers to stdout.
269  bool CompareVideosAndPrintResult(
270      int width,
271      int height,
272      const base::FilePath& captured_video_filename,
273      const base::FilePath& reference_video_filename,
274      const base::FilePath& stats_file) {
275
276    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
277        GetBrowserDir().Append(kFrameAnalyzerExecutable));
278    base::FilePath path_to_compare_script = GetSourceDir().Append(
279        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
280
281    if (!base::PathExists(path_to_analyzer)) {
282      LOG(ERROR) << "Missing frame analyzer: should be in "
283          << path_to_analyzer.value();
284      return false;
285    }
286    if (!base::PathExists(path_to_compare_script)) {
287      LOG(ERROR) << "Missing video compare script: should be in "
288          << path_to_compare_script.value();
289      return false;
290    }
291
292    // Note: don't append switches to this command since it will mess up the
293    // -u in the python invocation!
294    CommandLine compare_command(CommandLine::NO_PROGRAM);
295    EXPECT_TRUE(GetPythonCommand(&compare_command));
296
297    compare_command.AppendArgPath(path_to_compare_script);
298    compare_command.AppendArg("--label=VGA");
299    compare_command.AppendArg("--ref_video");
300    compare_command.AppendArgPath(reference_video_filename);
301    compare_command.AppendArg("--test_video");
302    compare_command.AppendArgPath(captured_video_filename);
303    compare_command.AppendArg("--frame_analyzer");
304    compare_command.AppendArgPath(path_to_analyzer);
305    compare_command.AppendArg("--yuv_frame_width");
306    compare_command.AppendArg(base::StringPrintf("%d", width));
307    compare_command.AppendArg("--yuv_frame_height");
308    compare_command.AppendArg(base::StringPrintf("%d", height));
309    compare_command.AppendArg("--stats_file");
310    compare_command.AppendArgPath(stats_file);
311
312    LOG(INFO) << "Running " << compare_command.GetCommandLineString();
313    std::string output;
314    bool ok = base::GetAppOutput(compare_command, &output);
315    // Print to stdout to ensure the perf numbers are parsed properly by the
316    // buildbot step.
317    printf("Output was:\n\n%s\n", output.c_str());
318    return ok;
319  }
320
321  base::FilePath GetWorkingDir() {
322    std::string home_dir;
323    environment_->GetVar(kHomeEnvName, &home_dir);
324    base::FilePath::StringType native_home_dir(home_dir.begin(),
325                                               home_dir.end());
326    return base::FilePath(native_home_dir).Append(kWorkingDirName);
327  }
328
329  PeerConnectionServerRunner peerconnection_server_;
330
331 private:
332  base::FilePath GetSourceDir() {
333    base::FilePath source_dir;
334    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
335    return source_dir;
336  }
337
338  base::FilePath GetBrowserDir() {
339    base::FilePath browser_dir;
340    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
341    return browser_dir;
342  }
343
344  base::ProcessHandle pywebsocket_server_;
345  scoped_ptr<base::Environment> environment_;
346};
347
348IN_PROC_BROWSER_TEST_F(WebrtcVideoQualityBrowserTest,
349                       MANUAL_TestVGAVideoQuality) {
350  ASSERT_TRUE(HasAllRequiredResources());
351  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
352  ASSERT_TRUE(StartPyWebSocketServer());
353  ASSERT_TRUE(peerconnection_server_.Start());
354
355  ui_test_utils::NavigateToURL(
356      browser(), embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage));
357  content::WebContents* left_tab =
358      browser()->tab_strip_model()->GetActiveWebContents();
359  GetUserMediaAndAccept(left_tab);
360
361  chrome::AddBlankTabAt(browser(), -1, true);
362  content::WebContents* right_tab =
363      browser()->tab_strip_model()->GetActiveWebContents();
364  ui_test_utils::NavigateToURL(
365      browser(), embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage));
366  GetUserMediaAndAccept(right_tab);
367
368  ConnectToPeerConnectionServer("peer 1", left_tab);
369  ConnectToPeerConnectionServer("peer 2", right_tab);
370
371  EstablishCall(left_tab, right_tab);
372
373  AssertNoAsynchronousErrors(left_tab);
374  AssertNoAsynchronousErrors(right_tab);
375
376  // Poll slower here to avoid flooding the log with messages: capturing and
377  // sending frames take quite a bit of time.
378  int polling_interval_msec = 1000;
379
380  EXPECT_TRUE(PollingWaitUntil(
381      "doneFrameCapturing()", "done-capturing", right_tab,
382      polling_interval_msec));
383
384  HangUp(left_tab);
385  WaitUntilHangupVerified(left_tab);
386  WaitUntilHangupVerified(right_tab);
387
388  AssertNoAsynchronousErrors(left_tab);
389  AssertNoAsynchronousErrors(right_tab);
390
391  EXPECT_TRUE(PollingWaitUntil(
392      "haveMoreFramesToSend()", "no-more-frames", right_tab,
393      polling_interval_msec));
394
395  RunARGBtoI420Converter(
396      kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName));
397  ASSERT_TRUE(
398      CompareVideosAndPrintResult(kVgaWidth,
399                                  kVgaHeight,
400                                  GetWorkingDir().Append(kCapturedYuvFileName),
401                                  GetWorkingDir().Append(kReferenceYuvFileName),
402                                  GetWorkingDir().Append(kStatsFileName)));
403
404  ASSERT_TRUE(peerconnection_server_.Stop());
405  ASSERT_TRUE(ShutdownPyWebSocketServer());
406}
407