chrome_webrtc_video_quality_browsertest.cc revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/environment.h"
6#include "base/file_util.h"
7#include "base/path_service.h"
8#include "base/process/launch.h"
9#include "base/strings/string_split.h"
10#include "base/strings/stringprintf.h"
11#include "base/test/test_timeouts.h"
12#include "base/time/time.h"
13#include "chrome/browser/chrome_notification_types.h"
14#include "chrome/browser/infobars/infobar.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "chrome/test/ui/ui_test.h"
27#include "content/public/browser/notification_service.h"
28#include "content/public/test/browser_test_utils.h"
29#include "media/base/media_switches.h"
30#include "net/test/embedded_test_server/embedded_test_server.h"
31#include "net/test/python_utils.h"
32#include "testing/perf/perf_test.h"
33#include "ui/gl/gl_switches.h"
34
35static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
36#if defined(OS_WIN)
37    FILE_PATH_LITERAL("frame_analyzer.exe");
38#else
39    FILE_PATH_LITERAL("frame_analyzer");
40#endif
41
42static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
43#if defined(OS_WIN)
44    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
45#else
46    FILE_PATH_LITERAL("rgba_to_i420_converter");
47#endif
48
49static const char kHomeEnvName[] =
50#if defined(OS_WIN)
51    "HOMEPATH";
52#else
53    "HOME";
54#endif
55
56// The working dir should be in the user's home folder.
57static const base::FilePath::CharType kWorkingDirName[] =
58    FILE_PATH_LITERAL("webrtc_video_quality");
59static const base::FilePath::CharType kReferenceYuvFileName[] =
60    FILE_PATH_LITERAL("reference_video.yuv");
61static const base::FilePath::CharType kCapturedYuvFileName[] =
62    FILE_PATH_LITERAL("captured_video.yuv");
63static const base::FilePath::CharType kStatsFileName[] =
64    FILE_PATH_LITERAL("stats.txt");
65static const char kMainWebrtcTestHtmlPage[] =
66    "/webrtc/webrtc_jsep01_test.html";
67static const char kCapturingWebrtcHtmlPage[] =
68    "/webrtc/webrtc_video_quality_test.html";
69static const int kVgaWidth = 640;
70static const int kVgaHeight = 480;
71
72// If you change the port number, don't forget to modify video_extraction.js
73// too!
74static const char kPyWebSocketPortNumber[] = "12221";
75
76// Test the video quality of the WebRTC output.
77//
78// Prerequisites: This test case must run on a machine with a virtual webcam
79// that plays video from the reference file located in <the running user's home
80// folder>/kWorkingDirName/kReferenceYuvFileName.
81//
82// You must also compile the chromium_builder_webrtc target before you run this
83// test to get all the tools built.
84//
85// The external compare_videos.py script also depends on two external
86// executables which must be located in the PATH when running this test.
87// * zxing (see the CPP version at https://code.google.com/p/zxing)
88// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
89//
90// The test case will launch a custom binary (peerconnection_server) which will
91// allow two WebRTC clients to find each other.
92//
93// The test also runs several other custom binaries - rgba_to_i420 converter and
94// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
95// test also runs a stand alone Python implementation of a WebSocket server
96// (pywebsocket) and a barcode_decoder script.
97class WebRtcVideoQualityBrowserTest : public WebRtcTestBase {
98 public:
99  WebRtcVideoQualityBrowserTest()
100      : pywebsocket_server_(0),
101        environment_(base::Environment::Create()) {}
102
103  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
104    PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem();
105    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
106  }
107
108  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
109    // This test expects real device handling and requires a real webcam / audio
110    // device; it will not work with fake devices.
111    EXPECT_FALSE(
112        command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream))
113        << "You cannot run this test with fake devices.";
114
115#if defined(OS_MACOSX)
116    // TODO(mcasas): Remove this switch when ManyCam virtual video capture
117    // device starts supporting AVFoundation, see http://crbug.com/327618.
118    command_line->AppendSwitch(switches::kDisableAVFoundation);
119#endif
120
121    // The video playback will not work without a GPU, so force its use here.
122    command_line->AppendSwitch(switches::kUseGpuInTests);
123  }
124
125  bool HasAllRequiredResources() {
126    if (!base::PathExists(GetWorkingDir())) {
127      LOG(ERROR) << "Cannot find the working directory for the reference video "
128          "and the temporary files:" << GetWorkingDir().value();
129      return false;
130    }
131    base::FilePath reference_file =
132        GetWorkingDir().Append(kReferenceYuvFileName);
133    if (!base::PathExists(reference_file)) {
134      LOG(ERROR) << "Cannot find the reference file to be used for video "
135          << "quality comparison: " << reference_file.value();
136      return false;
137    }
138    return true;
139  }
140
141  bool StartPyWebSocketServer() {
142    base::FilePath path_pywebsocket_dir =
143        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
144    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
145        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
146    base::FilePath path_to_data_handler =
147        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));
148
149    if (!base::PathExists(pywebsocket_server)) {
150      LOG(ERROR) << "Missing pywebsocket server.";
151      return false;
152    }
153    if (!base::PathExists(path_to_data_handler)) {
154      LOG(ERROR) << "Missing data handler for pywebsocket server.";
155      return false;
156    }
157
158    AppendToPythonPath(path_pywebsocket_dir);
159
160    // Note: don't append switches to this command since it will mess up the
161    // -u in the python invocation!
162    CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
163    EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
164
165    pywebsocket_command.AppendArgPath(pywebsocket_server);
166    pywebsocket_command.AppendArg("-p");
167    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
168    pywebsocket_command.AppendArg("-d");
169    pywebsocket_command.AppendArgPath(path_to_data_handler);
170
171    VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
172    return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
173                               &pywebsocket_server_);
174  }
175
176  bool ShutdownPyWebSocketServer() {
177    return base::KillProcess(pywebsocket_server_, 0, false);
178  }
179
180  void EstablishCall(content::WebContents* from_tab,
181                     content::WebContents* to_tab) {
182    EXPECT_EQ("ok-peerconnection-created",
183              ExecuteJavascript("preparePeerConnection()", from_tab));
184    EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab));
185    EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab));
186
187    // Ensure the call gets up on both sides.
188    EXPECT_TRUE(PollingWaitUntil(
189        "getPeerConnectionReadyState()", "active", from_tab));
190    EXPECT_TRUE(PollingWaitUntil(
191        "getPeerConnectionReadyState()", "active", to_tab));
192  }
193
194  void HangUp(content::WebContents* from_tab) {
195    EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
196  }
197
198  void WaitUntilHangupVerified(content::WebContents* tab_contents) {
199    EXPECT_TRUE(PollingWaitUntil(
200        "getPeerConnectionReadyState()", "no-peer-connection", tab_contents));
201  }
202
203  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
204  // which should contain frames of size |width| x |height|.
205  //
206  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
207  // should be build prior to running this test. The resulting binary should
208  // live next to Chrome.
209  bool RunARGBtoI420Converter(int width,
210                              int height,
211                              const base::FilePath& captured_video_filename) {
212    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
213        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
214
215    if (!base::PathExists(path_to_converter)) {
216      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
217          << path_to_converter.value();
218      return false;
219    }
220
221    CommandLine converter_command(path_to_converter);
222    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
223    converter_command.AppendSwitchPath("--output_file",
224                                       captured_video_filename);
225    converter_command.AppendSwitchASCII("--width",
226                                        base::StringPrintf("%d", width));
227    converter_command.AppendSwitchASCII("--height",
228                                        base::StringPrintf("%d", height));
229
230    // We produce an output file that will later be used as an input to the
231    // barcode decoder and frame analyzer tools.
232    VLOG(0) << "Running " << converter_command.GetCommandLineString();
233    std::string result;
234    bool ok = base::GetAppOutput(converter_command, &result);
235    VLOG(0) << "Output was:\n\n" << result;
236    return ok;
237  }
238
239  // Compares the |captured_video_filename| with the |reference_video_filename|.
240  //
241  // The barcode decoder decodes the captured video containing barcodes overlaid
242  // into every frame of the video (produced by rgba_to_i420_converter). It
243  // produces a set of PNG images and a |stats_file| that maps each captured
244  // frame to a frame in the reference video. The frames should be of size
245  // |width| x |height|.
246  // All measurements calculated are printed as perf parsable numbers to stdout.
247  bool CompareVideosAndPrintResult(
248      int width,
249      int height,
250      const base::FilePath& captured_video_filename,
251      const base::FilePath& reference_video_filename,
252      const base::FilePath& stats_file) {
253
254    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
255        GetBrowserDir().Append(kFrameAnalyzerExecutable));
256    base::FilePath path_to_compare_script = GetSourceDir().Append(
257        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
258
259    if (!base::PathExists(path_to_analyzer)) {
260      LOG(ERROR) << "Missing frame analyzer: should be in "
261          << path_to_analyzer.value();
262      return false;
263    }
264    if (!base::PathExists(path_to_compare_script)) {
265      LOG(ERROR) << "Missing video compare script: should be in "
266          << path_to_compare_script.value();
267      return false;
268    }
269
270    // Note: don't append switches to this command since it will mess up the
271    // -u in the python invocation!
272    CommandLine compare_command(CommandLine::NO_PROGRAM);
273    EXPECT_TRUE(GetPythonCommand(&compare_command));
274
275    compare_command.AppendArgPath(path_to_compare_script);
276    compare_command.AppendArg("--label=VGA");
277    compare_command.AppendArg("--ref_video");
278    compare_command.AppendArgPath(reference_video_filename);
279    compare_command.AppendArg("--test_video");
280    compare_command.AppendArgPath(captured_video_filename);
281    compare_command.AppendArg("--frame_analyzer");
282    compare_command.AppendArgPath(path_to_analyzer);
283    compare_command.AppendArg("--yuv_frame_width");
284    compare_command.AppendArg(base::StringPrintf("%d", width));
285    compare_command.AppendArg("--yuv_frame_height");
286    compare_command.AppendArg(base::StringPrintf("%d", height));
287    compare_command.AppendArg("--stats_file");
288    compare_command.AppendArgPath(stats_file);
289
290    VLOG(0) << "Running " << compare_command.GetCommandLineString();
291    std::string output;
292    bool ok = base::GetAppOutput(compare_command, &output);
293    // Print to stdout to ensure the perf numbers are parsed properly by the
294    // buildbot step.
295    printf("Output was:\n\n%s\n", output.c_str());
296    return ok;
297  }
298
299  base::FilePath GetWorkingDir() {
300    std::string home_dir;
301    environment_->GetVar(kHomeEnvName, &home_dir);
302    base::FilePath::StringType native_home_dir(home_dir.begin(),
303                                               home_dir.end());
304    return base::FilePath(native_home_dir).Append(kWorkingDirName);
305  }
306
307  PeerConnectionServerRunner peerconnection_server_;
308
309 private:
310  base::FilePath GetSourceDir() {
311    base::FilePath source_dir;
312    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
313    return source_dir;
314  }
315
316  base::FilePath GetBrowserDir() {
317    base::FilePath browser_dir;
318    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
319    return browser_dir;
320  }
321
322  base::ProcessHandle pywebsocket_server_;
323  scoped_ptr<base::Environment> environment_;
324};
325
326IN_PROC_BROWSER_TEST_F(WebRtcVideoQualityBrowserTest,
327                       MANUAL_TestVGAVideoQuality) {
328  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
329      "This is a long-running test; you must specify "
330      "--ui-test-action-max-timeout to have a value of at least 150000.";
331
332  ASSERT_TRUE(HasAllRequiredResources());
333  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
334  ASSERT_TRUE(StartPyWebSocketServer());
335  ASSERT_TRUE(peerconnection_server_.Start());
336
337  content::WebContents* left_tab =
338      OpenPageAndGetUserMediaInNewTab(
339          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage));
340  content::WebContents* right_tab =
341      OpenPageAndGetUserMediaInNewTab(
342          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage));
343
344  ConnectToPeerConnectionServer("peer 1", left_tab);
345  ConnectToPeerConnectionServer("peer 2", right_tab);
346
347  EstablishCall(left_tab, right_tab);
348
349  // Poll slower here to avoid flooding the log with messages: capturing and
350  // sending frames take quite a bit of time.
351  int polling_interval_msec = 1000;
352
353  EXPECT_TRUE(PollingWaitUntil(
354      "doneFrameCapturing()", "done-capturing", right_tab,
355      polling_interval_msec));
356
357  HangUp(left_tab);
358  WaitUntilHangupVerified(left_tab);
359  WaitUntilHangupVerified(right_tab);
360
361  EXPECT_TRUE(PollingWaitUntil(
362      "haveMoreFramesToSend()", "no-more-frames", right_tab,
363      polling_interval_msec));
364
365  // Shut everything down to avoid having the javascript race with the analysis
366  // tools. For instance, dont have console log printouts interleave with the
367  // RESULT lines from the analysis tools (crbug.com/323200).
368  ASSERT_TRUE(peerconnection_server_.Stop());
369  ASSERT_TRUE(ShutdownPyWebSocketServer());
370
371  chrome::CloseWebContents(browser(), left_tab, false);
372  chrome::CloseWebContents(browser(), right_tab, false);
373
374  RunARGBtoI420Converter(
375      kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName));
376  ASSERT_TRUE(
377      CompareVideosAndPrintResult(kVgaWidth,
378                                  kVgaHeight,
379                                  GetWorkingDir().Append(kCapturedYuvFileName),
380                                  GetWorkingDir().Append(kReferenceYuvFileName),
381                                  GetWorkingDir().Append(kStatsFileName)));
382}
383