chrome_webrtc_video_quality_browsertest.cc revision 558790d6acca3451cf3a6b497803a5f07d0bec58 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/environment.h"
6#include "base/file_util.h"
7#include "base/path_service.h"
8#include "base/process/launch.h"
9#include "base/strings/string_split.h"
10#include "base/strings/stringprintf.h"
11#include "base/test/test_timeouts.h"
12#include "base/time/time.h"
13#include "chrome/browser/chrome_notification_types.h"
14#include "chrome/browser/infobars/infobar.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "chrome/test/perf/perf_test.h"
27#include "chrome/test/ui/ui_test.h"
28#include "content/public/browser/notification_service.h"
29#include "content/public/test/browser_test_utils.h"
30#include "net/test/python_utils.h"
31#include "net/test/spawned_test_server/spawned_test_server.h"
32
33static const base::FilePath::CharType kPeerConnectionServer[] =
34#if defined(OS_WIN)
35    FILE_PATH_LITERAL("peerconnection_server.exe");
36#else
37    FILE_PATH_LITERAL("peerconnection_server");
38#endif
39
40static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
41#if defined(OS_WIN)
42    FILE_PATH_LITERAL("frame_analyzer.exe");
43#else
44    FILE_PATH_LITERAL("frame_analyzer");
45#endif
46
47static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
48#if defined(OS_WIN)
49    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
50#else
51    FILE_PATH_LITERAL("rgba_to_i420_converter");
52#endif
53
54static const char kHomeEnvName[] =
55#if defined(OS_WIN)
56    "HOMEPATH";
57#else
58    "HOME";
59#endif
60
61// The working dir should be in the user's home folder.
62static const base::FilePath::CharType kWorkingDirName[] =
63    FILE_PATH_LITERAL("webrtc_video_quality");
64static const base::FilePath::CharType kReferenceYuvFileName[] =
65    FILE_PATH_LITERAL("reference_video.yuv");
66static const base::FilePath::CharType kCapturedYuvFileName[] =
67    FILE_PATH_LITERAL("captured_video.yuv");
68static const base::FilePath::CharType kStatsFileName[] =
69    FILE_PATH_LITERAL("stats.txt");
70static const char kMainWebrtcTestHtmlPage[] =
71    "files/webrtc/webrtc_jsep01_test.html";
72static const char kCapturingWebrtcHtmlPage[] =
73    "files/webrtc/webrtc_video_quality_test.html";
74static const int kVgaWidth = 640;
75static const int kVgaHeight = 480;
76
77// If you change the port number, don't forget to modify video_extraction.js
78// too!
79static const char kPyWebSocketPortNumber[] = "12221";
80
81// Test the video quality of the WebRTC output.
82//
83// Prerequisites: This test case must run on a machine with a virtual webcam
84// that plays video from the reference file located in <the running users home
85// folder>/kWorkingDirName/kReferenceYuvFileName.
86//
87// You must also compile the chromium_builder_webrtc target before you run this
88// test to get all the tools built.
89//
90// The external compare_videos.py script also depends on two external
91// executables which must be located in the PATH when running this test.
92// * zxing (see the CPP version at https://code.google.com/p/zxing)
93// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
94//
95// The test case will launch a custom binary (peerconnection_server) which will
96// allow two WebRTC clients to find each other.
97//
98// The test also runs several other custom binaries - rgba_to_i420 converter and
99// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
100// test also runs a stand alone Python implementation of a WebSocket server
101// (pywebsocket) and a barcode_decoder script.
102class WebrtcVideoQualityBrowserTest : public WebRtcTestBase {
103 public:
104  WebrtcVideoQualityBrowserTest()
105      : peerconnection_server_(0),
106        pywebsocket_server_(0),
107        environment_(base::Environment::Create()) {}
108
109  virtual void SetUp() OVERRIDE {
110    RunPeerConnectionServer();
111    InProcessBrowserTest::SetUp();
112
113    // Ensure we have the stuff we need.
114    EXPECT_TRUE(base::PathExists(GetWorkingDir()))
115        << "Cannot find the working directory for the reference video and "
116           "the temporary files:" << GetWorkingDir().value();
117    base::FilePath reference_file =
118        GetWorkingDir().Append(kReferenceYuvFileName);
119    EXPECT_TRUE(base::PathExists(reference_file))
120        << "Cannot find the reference file to be used for video quality "
121        << "comparison: " << reference_file.value();
122  }
123
124  virtual void TearDown() OVERRIDE {
125    ShutdownPeerConnectionServer();
126    InProcessBrowserTest::TearDown();
127    ShutdownPyWebSocketServer();
128  }
129
130  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
131    // TODO(phoglund): check that user actually has the requisite devices and
132    // print a nice message if not; otherwise the test just times out which can
133    // be confusing.
134    // This test expects real device handling and requires a real webcam / audio
135    // device; it will not work with fake devices.
136    EXPECT_FALSE(
137        command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream))
138        << "You cannot run this test with fake devices.";
139  }
140
141  void StartPyWebSocketServer() {
142    base::FilePath path_pywebsocket_dir =
143        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
144    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
145        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
146    base::FilePath path_to_data_handler =
147        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));
148
149    EXPECT_TRUE(base::PathExists(pywebsocket_server))
150        << "Fatal: missing pywebsocket server.";
151    EXPECT_TRUE(base::PathExists(path_to_data_handler))
152        << "Fatal: missing data handler for pywebsocket server.";
153
154    AppendToPythonPath(path_pywebsocket_dir);
155    CommandLine pywebsocket_command = MakePythonCommand(pywebsocket_server);
156
157    // Construct the command line manually, the server doesn't support -arg=val.
158    pywebsocket_command.AppendArg("-p");
159    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
160    pywebsocket_command.AppendArg("-d");
161    pywebsocket_command.AppendArgPath(path_to_data_handler);
162
163    LOG(INFO) << "Running " << pywebsocket_command.GetCommandLineString();
164    EXPECT_TRUE(base::LaunchProcess(
165        pywebsocket_command, base::LaunchOptions(), &pywebsocket_server_))
166        << "Failed to launch pywebsocket server.";
167  }
168
169  void ShutdownPyWebSocketServer() {
170    EXPECT_TRUE(base::KillProcess(pywebsocket_server_, 0, false))
171        << "Failed to shut down pywebsocket server!";
172  }
173
174  // Convenience method which executes the provided javascript in the context
175  // of the provided web contents and returns what it evaluated to.
176  std::string ExecuteJavascript(const std::string& javascript,
177                                content::WebContents* tab_contents) {
178    std::string result;
179    EXPECT_TRUE(content::ExecuteScriptAndExtractString(
180        tab_contents, javascript, &result));
181    return result;
182  }
183
184  // Ensures we didn't get any errors asynchronously (e.g. while no javascript
185  // call from this test was outstanding).
186  // TODO(phoglund): this becomes obsolete when we switch to communicating with
187  // the DOM message queue.
188  void AssertNoAsynchronousErrors(content::WebContents* tab_contents) {
189    EXPECT_EQ("ok-no-errors",
190              ExecuteJavascript("getAnyTestFailures()", tab_contents));
191  }
192
193  // The peer connection server lets our two tabs find each other and talk to
194  // each other (e.g. it is the application-specific "signaling solution").
195  void ConnectToPeerConnectionServer(const std::string peer_name,
196                                     content::WebContents* tab_contents) {
197    std::string javascript = base::StringPrintf(
198        "connect('http://localhost:8888', '%s');", peer_name.c_str());
199    EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents));
200  }
201
202  void EstablishCall(content::WebContents* from_tab,
203                     content::WebContents* to_tab) {
204    EXPECT_EQ("ok-peerconnection-created",
205              ExecuteJavascript("preparePeerConnection()", from_tab));
206    EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab));
207    EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab));
208
209    // Ensure the call gets up on both sides.
210    EXPECT_TRUE(PollingWaitUntil(
211        "getPeerConnectionReadyState()", "active", from_tab));
212    EXPECT_TRUE(PollingWaitUntil(
213        "getPeerConnectionReadyState()", "active", to_tab));
214  }
215
216  void HangUp(content::WebContents* from_tab) {
217    EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
218  }
219
220  void WaitUntilHangupVerified(content::WebContents* tab_contents) {
221    EXPECT_TRUE(PollingWaitUntil(
222        "getPeerConnectionReadyState()", "no-peer-connection", tab_contents));
223  }
224
225  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
226  // which should contain frames of size |width| x |height|.
227  //
228  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
229  // should be build prior to running this test. The resulting binary should
230  // live next to Chrome.
231  void RunARGBtoI420Converter(int width,
232                              int height,
233                              const base::FilePath& captured_video_filename) {
234    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
235        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
236    EXPECT_TRUE(base::PathExists(path_to_converter))
237        << "Missing ARGB->I420 converter: should be in "
238        << path_to_converter.value();
239
240    CommandLine converter_command(path_to_converter);
241    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
242    converter_command.AppendSwitchPath("--output_file",
243                                       captured_video_filename);
244    converter_command.AppendSwitchASCII("--width",
245                                        base::StringPrintf("%d", width));
246    converter_command.AppendSwitchASCII("--height",
247                                        base::StringPrintf("%d", height));
248
249    // We produce an output file that will later be used as an input to the
250    // barcode decoder and frame analyzer tools.
251    LOG(INFO) << "Running " << converter_command.GetCommandLineString();
252    std::string result;
253    EXPECT_TRUE(base::GetAppOutput(converter_command, &result));
254    LOG(INFO) << "Output was:\n\n" << result;
255  }
256
257  // Compares the |captured_video_filename| with the |reference_video_filename|.
258  //
259  // The barcode decoder decodes the captured video containing barcodes overlaid
260  // into every frame of the video (produced by rgba_to_i420_converter). It
261  // produces a set of PNG images and a |stats_file| that maps each captured
262  // frame to a frame in the reference video. The frames should be of size
263  // |width| x |height|. The output of compare_videos.py is returned.
264  std::string CompareVideos(int width,
265                            int height,
266                            const base::FilePath& captured_video_filename,
267                            const base::FilePath& reference_video_filename,
268                            const base::FilePath& stats_file) {
269    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
270        GetBrowserDir().Append(kFrameAnalyzerExecutable));
271    base::FilePath path_to_compare_script = GetSourceDir().Append(
272        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
273
274    EXPECT_TRUE(base::PathExists(path_to_analyzer))
275        << "Missing frame analyzer: should be in " << path_to_analyzer.value();
276    EXPECT_TRUE(base::PathExists(path_to_compare_script))
277        << "Missing video compare script: should be in "
278        << path_to_compare_script.value();
279
280    CommandLine compare_command = MakePythonCommand(path_to_compare_script);
281    compare_command.AppendSwitchPath("--ref_video", reference_video_filename);
282    compare_command.AppendSwitchPath("--test_video", captured_video_filename);
283    compare_command.AppendSwitchPath("--frame_analyzer", path_to_analyzer);
284    compare_command.AppendSwitchASCII("--yuv_frame_width",
285                                      base::StringPrintf("%d", width));
286    compare_command.AppendSwitchASCII("--yuv_frame_height",
287                                      base::StringPrintf("%d", height));
288    compare_command.AppendSwitchPath("--stats_file", stats_file);
289
290    LOG(INFO) << "Running " << compare_command.GetCommandLineString();
291    std::string result;
292    EXPECT_TRUE(base::GetAppOutput(compare_command, &result));
293    LOG(INFO) << "Output was:\n\n" << result;
294    return result;
295  }
296
297  // Processes the |frame_analyzer_output| for the different frame counts.
298  //
299  // The frame analyzer outputs additional information about the number of
300  // unique frames captured, The max number of repeated frames in a sequence and
301  // the max number of skipped frames. These values are then written to the Perf
302  // Graph. (Note: Some of the repeated or skipped frames will probably be due
303  // to the imperfection of JavaScript timers).
304  void PrintFramesCountPerfResults(std::string frame_analyzer_output) {
305    size_t unique_frames_pos =
306        frame_analyzer_output.rfind("Unique_frames_count");
307    EXPECT_NE(unique_frames_pos, std::string::npos)
308        << "Missing Unique_frames_count in frame analyzer output:\n"
309        << frame_analyzer_output;
310
311    std::string unique_frame_counts =
312        frame_analyzer_output.substr(unique_frames_pos);
313    // TODO(phoglund): Fix ESTATS result to not have this silly newline.
314    std::replace(
315        unique_frame_counts.begin(), unique_frame_counts.end(), '\n', ' ');
316
317    std::vector<std::pair<std::string, std::string> > key_values;
318    base::SplitStringIntoKeyValuePairs(
319        unique_frame_counts, ':', ' ', &key_values);
320    std::vector<std::pair<std::string, std::string> >::const_iterator iter;
321    for (iter = key_values.begin(); iter != key_values.end(); ++iter) {
322      const std::pair<std::string, std::string>& key_value = *iter;
323      perf_test::PrintResult(
324          key_value.first, "", "VGA", key_value.second, "", false);
325    }
326  }
327
328  // Processes the |frame_analyzer_output| to extract the PSNR and SSIM values.
329  //
330  // The frame analyzer produces PSNR and SSIM results for every unique frame
331  // that has been captured. This method forms a list of all the psnr and ssim
332  // values and passes it to PrintResultList() for printing on the Perf Graph.
333  void PrintPsnrAndSsimPerfResults(std::string frame_analyzer_output) {
334    size_t stats_start = frame_analyzer_output.find("BSTATS");
335    EXPECT_NE(stats_start, std::string::npos)
336        << "Missing BSTATS in frame analyzer output:\n"
337        << frame_analyzer_output;
338    size_t stats_end = frame_analyzer_output.find("ESTATS");
339    EXPECT_NE(stats_end, std::string::npos)
340        << "Missing ESTATS in frame analyzer output:\n"
341        << frame_analyzer_output;
342
343    stats_start += std::string("BSTATS").size();
344    std::string psnr_ssim_stats =
345        frame_analyzer_output.substr(stats_start, stats_end - stats_start);
346
347    // PSNR and SSIM values aren't really key-value pairs but it is convenient
348    // to parse them as such.
349    // TODO(phoglund): make the format more convenient so we need less
350    // processing here.
351    std::vector<std::pair<std::string, std::string> > psnr_ssim_entries;
352    base::SplitStringIntoKeyValuePairs(
353        psnr_ssim_stats, ' ', ';', &psnr_ssim_entries);
354
355    std::string psnr_value_list;
356    std::string ssim_value_list;
357    std::vector<std::pair<std::string, std::string> >::const_iterator iter;
358    for (iter = psnr_ssim_entries.begin(); iter != psnr_ssim_entries.end();
359         ++iter) {
360      const std::pair<std::string, std::string>& psnr_and_ssim = *iter;
361      psnr_value_list.append(psnr_and_ssim.first).append(",");
362      ssim_value_list.append(psnr_and_ssim.second).append(",");
363    }
364    // Nuke last comma.
365    psnr_value_list.erase(psnr_value_list.size() - 1);
366    ssim_value_list.erase(ssim_value_list.size() - 1);
367
368    perf_test::PrintResultList("PSNR", "", "VGA", psnr_value_list, "dB", false);
369    perf_test::PrintResultList("SSIM", "", "VGA", ssim_value_list, "", false);
370  }
371
372  base::FilePath GetWorkingDir() {
373    std::string home_dir;
374    environment_->GetVar(kHomeEnvName, &home_dir);
375    base::FilePath::StringType native_home_dir(home_dir.begin(),
376                                               home_dir.end());
377    return base::FilePath(native_home_dir).Append(kWorkingDirName);
378  }
379
380 private:
381  void RunPeerConnectionServer() {
382    // TODO(phoglund): de-dupe later: next line differs from original.
383    base::FilePath peerconnection_server =
384        GetBrowserDir().Append(kPeerConnectionServer);
385
386    EXPECT_TRUE(base::PathExists(peerconnection_server))
387        << "Missing peerconnection_server. You must build "
388           "it so it ends up next to the browser test binary.";
389    EXPECT_TRUE(base::LaunchProcess(CommandLine(peerconnection_server),
390                                    base::LaunchOptions(),
391                                    &peerconnection_server_))
392        << "Failed to launch peerconnection_server.";
393  }
394
395  void ShutdownPeerConnectionServer() {
396    EXPECT_TRUE(base::KillProcess(peerconnection_server_, 0, false))
397        << "Failed to shut down peerconnection_server!";
398  }
399
400  base::FilePath GetSourceDir() {
401    base::FilePath source_dir;
402    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
403    return source_dir;
404  }
405
406  base::FilePath GetBrowserDir() {
407    base::FilePath browser_dir;
408    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
409    return browser_dir;
410  }
411
412  CommandLine MakePythonCommand(base::FilePath python_script) {
413    CommandLine python_command(CommandLine::NO_PROGRAM);
414    EXPECT_TRUE(GetPythonCommand(&python_command));
415    CommandLine complete_command(python_script);
416    complete_command.PrependWrapper(python_command.GetCommandLineString());
417    return complete_command;
418  }
419
420  base::ProcessHandle peerconnection_server_;
421  base::ProcessHandle pywebsocket_server_;
422  scoped_ptr<base::Environment> environment_;
423};
424
425#if defined(OS_WIN)
426// Broken on Win: failing to start pywebsocket_server. http://crbug.com/255499.
427#define MAYBE_MANUAL_TestVGAVideoQuality DISABLED_MANUAL_TestVGAVideoQuality
428#else
429#define MAYBE_MANUAL_TestVGAVideoQuality MANUAL_TestVGAVideoQuality
430#endif
431
432IN_PROC_BROWSER_TEST_F(WebrtcVideoQualityBrowserTest,
433                       MAYBE_MANUAL_TestVGAVideoQuality) {
434  // TODO(phoglund): de-dupe from chrome_webrtc_browsertest.cc.
435  StartPyWebSocketServer();
436
437  EXPECT_TRUE(test_server()->Start());
438
439  ui_test_utils::NavigateToURL(browser(),
440                               test_server()->GetURL(kMainWebrtcTestHtmlPage));
441  content::WebContents* left_tab =
442      browser()->tab_strip_model()->GetActiveWebContents();
443  GetUserMediaAndAccept(left_tab);
444
445  chrome::AddBlankTabAt(browser(), -1, true);
446  content::WebContents* right_tab =
447      browser()->tab_strip_model()->GetActiveWebContents();
448  // TODO(phoglund): (de-dupe later) different from original flow.
449  ui_test_utils::NavigateToURL(browser(),
450                               test_server()->GetURL(kCapturingWebrtcHtmlPage));
451  GetUserMediaAndAccept(right_tab);
452
453  ConnectToPeerConnectionServer("peer 1", left_tab);
454  ConnectToPeerConnectionServer("peer 2", right_tab);
455
456  EstablishCall(left_tab, right_tab);
457
458  AssertNoAsynchronousErrors(left_tab);
459  AssertNoAsynchronousErrors(right_tab);
460
461  // Poll slower here to avoid flooding the log with messages: capturing and
462  // sending frames take quite a bit of time.
463  int polling_interval_msec = 1000;
464
465  // TODO(phoglund): (de-dupe later) different from original flow.
466  EXPECT_TRUE(PollingWaitUntil(
467      "doneFrameCapturing()", "done-capturing", right_tab,
468      polling_interval_msec));
469
470  HangUp(left_tab);
471  WaitUntilHangupVerified(left_tab);
472  WaitUntilHangupVerified(right_tab);
473
474  AssertNoAsynchronousErrors(left_tab);
475  AssertNoAsynchronousErrors(right_tab);
476
477  // TODO(phoglund): (de-dupe later) different from original flow.
478  EXPECT_TRUE(PollingWaitUntil(
479      "haveMoreFramesToSend()", "no-more-frames", right_tab,
480      polling_interval_msec));
481
482  RunARGBtoI420Converter(
483      kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName));
484  std::string output =
485      CompareVideos(kVgaWidth,
486                    kVgaHeight,
487                    GetWorkingDir().Append(kCapturedYuvFileName),
488                    GetWorkingDir().Append(kReferenceYuvFileName),
489                    GetWorkingDir().Append(kStatsFileName));
490
491  PrintFramesCountPerfResults(output);
492  PrintPsnrAndSsimPerfResults(output);
493}
494