chrome_webrtc_video_quality_browsertest.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/environment.h"
6#include "base/file_util.h"
7#include "base/path_service.h"
8#include "base/process/launch.h"
9#include "base/strings/string_split.h"
10#include "base/strings/stringprintf.h"
11#include "base/test/test_timeouts.h"
12#include "base/time/time.h"
13#include "chrome/browser/chrome_notification_types.h"
14#include "chrome/browser/infobars/infobar.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "chrome/test/ui/ui_test.h"
27#include "content/public/browser/notification_service.h"
28#include "content/public/test/browser_test_utils.h"
29#include "media/base/media_switches.h"
30#include "net/test/embedded_test_server/embedded_test_server.h"
31#include "net/test/python_utils.h"
32#include "testing/perf/perf_test.h"
33#include "ui/gl/gl_switches.h"
34
35// For fine-grained suppression on flaky tests.
36#if defined(OS_WIN)
37#include "base/win/windows_version.h"
38#endif
39
40static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
41#if defined(OS_WIN)
42    FILE_PATH_LITERAL("frame_analyzer.exe");
43#else
44    FILE_PATH_LITERAL("frame_analyzer");
45#endif
46
47static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
48#if defined(OS_WIN)
49    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
50#else
51    FILE_PATH_LITERAL("rgba_to_i420_converter");
52#endif
53
54static const char kHomeEnvName[] =
55#if defined(OS_WIN)
56    "HOMEPATH";
57#else
58    "HOME";
59#endif
60
61// The working dir should be in the user's home folder.
62static const base::FilePath::CharType kWorkingDirName[] =
63    FILE_PATH_LITERAL("webrtc_video_quality");
64static const base::FilePath::CharType kReferenceVideosDirName[] =
65    FILE_PATH_LITERAL("webrtc.DEPS/webrtc_videos");
66static const base::FilePath::CharType kReferenceFileName360p[] =
67    FILE_PATH_LITERAL("reference_video_640x360_30fps");
68static const base::FilePath::CharType kYuvFileExtension[] =
69    FILE_PATH_LITERAL("yuv");
70static const base::FilePath::CharType kY4mFileExtension[] =
71    FILE_PATH_LITERAL("y4m");
72static const base::FilePath::CharType kCapturedYuvFileName[] =
73    FILE_PATH_LITERAL("captured_video.yuv");
74static const base::FilePath::CharType kStatsFileName[] =
75    FILE_PATH_LITERAL("stats.txt");
76static const char kMainWebrtcTestHtmlPage[] =
77    "/webrtc/webrtc_jsep01_test.html";
78static const char kCapturingWebrtcHtmlPage[] =
79    "/webrtc/webrtc_video_quality_test.html";
80static const int k360pWidth = 640;
81static const int k360pHeight = 360;
82
83// If you change the port number, don't forget to modify video_extraction.js
84// too!
85static const char kPyWebSocketPortNumber[] = "12221";
86
87const char kAdviseOnGclientSolution[] =
88    "You need to add this solution to your .gclient to run this test:\n"
89    "{\n"
90    "  \"name\"        : \"webrtc.DEPS\",\n"
91    "  \"url\"         : \"svn://svn.chromium.org/chrome/trunk/deps/"
92    "third_party/webrtc/webrtc.DEPS\",\n"
93    "}";
94
95// Test the video quality of the WebRTC output.
96//
97// Prerequisites: This test case must run on a machine with a chrome playing
98// the video from the reference files located int GetReferenceVideosDir().
99// The file kReferenceY4mFileName.kY4mFileExtension is played using a
100// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
101// comparison.
102//
103// You must also compile the chromium_builder_webrtc target before you run this
104// test to get all the tools built.
105//
106// The external compare_videos.py script also depends on two external
107// executables which must be located in the PATH when running this test.
108// * zxing (see the CPP version at https://code.google.com/p/zxing)
109// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
110//
111// The test case will launch a custom binary (peerconnection_server) which will
112// allow two WebRTC clients to find each other.
113//
114// The test also runs several other custom binaries - rgba_to_i420 converter and
115// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
116// test also runs a stand alone Python implementation of a WebSocket server
117// (pywebsocket) and a barcode_decoder script.
118class WebRtcVideoQualityBrowserTest : public WebRtcTestBase {
119 public:
120  WebRtcVideoQualityBrowserTest()
121      : pywebsocket_server_(0),
122        environment_(base::Environment::Create()) {}
123
124  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
125    PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem();
126    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
127  }
128
129  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
130    // Set up the command line option with the expected file name. We will check
131    // its existence in HasAllRequiredResources().
132    webrtc_reference_video_y4m_ = GetReferenceVideosDir()
133        .Append(kReferenceFileName360p).AddExtension(kY4mFileExtension);
134    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
135                                   webrtc_reference_video_y4m_);
136    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
137
138    // The video playback will not work without a GPU, so force its use here.
139    command_line->AppendSwitch(switches::kUseGpuInTests);
140  }
141
142  bool HasAllRequiredResources() {
143    if (!base::PathExists(GetWorkingDir())) {
144      LOG(ERROR) << "Cannot find the working directory for the temporary "
145          "files:" << GetWorkingDir().value();
146      return false;
147    }
148    if (!base::PathExists(GetReferenceVideosDir())) {
149      LOG(ERROR) << "Cannot find the working directory for the reference video "
150          "files, expected at " << GetReferenceVideosDir().value() << ". " <<
151          kAdviseOnGclientSolution;
152      return false;
153    }
154    base::FilePath webrtc_reference_video_yuv = GetReferenceVideosDir()
155        .Append(kReferenceFileName360p).AddExtension(kYuvFileExtension);
156    if (!base::PathExists(webrtc_reference_video_yuv)) {
157      LOG(ERROR) << "Missing YUV reference video to be used for quality"
158          << " comparison, expected at " << webrtc_reference_video_yuv.value()
159          << ". " << kAdviseOnGclientSolution;
160      return false;
161    }
162    if (!base::PathExists(webrtc_reference_video_y4m_)) {
163      LOG(ERROR) << "Missing Y4M reference video to be used for quality"
164          << " comparison, expected at "<< webrtc_reference_video_y4m_.value()
165          << ". " << kAdviseOnGclientSolution;
166      return false;
167    }
168    return true;
169  }
170
171  bool StartPyWebSocketServer() {
172    base::FilePath path_pywebsocket_dir =
173        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
174    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
175        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
176    base::FilePath path_to_data_handler =
177        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));
178
179    if (!base::PathExists(pywebsocket_server)) {
180      LOG(ERROR) << "Missing pywebsocket server.";
181      return false;
182    }
183    if (!base::PathExists(path_to_data_handler)) {
184      LOG(ERROR) << "Missing data handler for pywebsocket server.";
185      return false;
186    }
187
188    AppendToPythonPath(path_pywebsocket_dir);
189
190    // Note: don't append switches to this command since it will mess up the
191    // -u in the python invocation!
192    CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
193    EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
194
195    pywebsocket_command.AppendArgPath(pywebsocket_server);
196    pywebsocket_command.AppendArg("-p");
197    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
198    pywebsocket_command.AppendArg("-d");
199    pywebsocket_command.AppendArgPath(path_to_data_handler);
200
201    VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
202    return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
203                               &pywebsocket_server_);
204  }
205
206  bool ShutdownPyWebSocketServer() {
207    return base::KillProcess(pywebsocket_server_, 0, false);
208  }
209
210  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
211  // which should contain frames of size |width| x |height|.
212  //
213  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
214  // should be build prior to running this test. The resulting binary should
215  // live next to Chrome.
216  bool RunARGBtoI420Converter(int width,
217                              int height,
218                              const base::FilePath& captured_video_filename) {
219    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
220        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
221
222    if (!base::PathExists(path_to_converter)) {
223      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
224          << path_to_converter.value();
225      return false;
226    }
227
228    CommandLine converter_command(path_to_converter);
229    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
230    converter_command.AppendSwitchPath("--output_file",
231                                       captured_video_filename);
232    converter_command.AppendSwitchASCII("--width",
233                                        base::StringPrintf("%d", width));
234    converter_command.AppendSwitchASCII("--height",
235                                        base::StringPrintf("%d", height));
236
237    // We produce an output file that will later be used as an input to the
238    // barcode decoder and frame analyzer tools.
239    VLOG(0) << "Running " << converter_command.GetCommandLineString();
240    std::string result;
241    bool ok = base::GetAppOutput(converter_command, &result);
242    VLOG(0) << "Output was:\n\n" << result;
243    return ok;
244  }
245
246  // Compares the |captured_video_filename| with the |reference_video_filename|.
247  //
248  // The barcode decoder decodes the captured video containing barcodes overlaid
249  // into every frame of the video (produced by rgba_to_i420_converter). It
250  // produces a set of PNG images and a |stats_file| that maps each captured
251  // frame to a frame in the reference video. The frames should be of size
252  // |width| x |height|.
253  // All measurements calculated are printed as perf parsable numbers to stdout.
254  bool CompareVideosAndPrintResult(
255      int width,
256      int height,
257      const base::FilePath& captured_video_filename,
258      const base::FilePath& reference_video_filename,
259      const base::FilePath& stats_file) {
260
261    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
262        GetBrowserDir().Append(kFrameAnalyzerExecutable));
263    base::FilePath path_to_compare_script = GetSourceDir().Append(
264        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
265
266    if (!base::PathExists(path_to_analyzer)) {
267      LOG(ERROR) << "Missing frame analyzer: should be in "
268          << path_to_analyzer.value();
269      return false;
270    }
271    if (!base::PathExists(path_to_compare_script)) {
272      LOG(ERROR) << "Missing video compare script: should be in "
273          << path_to_compare_script.value();
274      return false;
275    }
276
277    // Note: don't append switches to this command since it will mess up the
278    // -u in the python invocation!
279    CommandLine compare_command(CommandLine::NO_PROGRAM);
280    EXPECT_TRUE(GetPythonCommand(&compare_command));
281
282    compare_command.AppendArgPath(path_to_compare_script);
283    compare_command.AppendArg("--label=360p");
284    compare_command.AppendArg("--ref_video");
285    compare_command.AppendArgPath(reference_video_filename);
286    compare_command.AppendArg("--test_video");
287    compare_command.AppendArgPath(captured_video_filename);
288    compare_command.AppendArg("--frame_analyzer");
289    compare_command.AppendArgPath(path_to_analyzer);
290    compare_command.AppendArg("--yuv_frame_width");
291    compare_command.AppendArg(base::StringPrintf("%d", width));
292    compare_command.AppendArg("--yuv_frame_height");
293    compare_command.AppendArg(base::StringPrintf("%d", height));
294    compare_command.AppendArg("--stats_file");
295    compare_command.AppendArgPath(stats_file);
296
297    VLOG(0) << "Running " << compare_command.GetCommandLineString();
298    std::string output;
299    bool ok = base::GetAppOutput(compare_command, &output);
300    // Print to stdout to ensure the perf numbers are parsed properly by the
301    // buildbot step.
302    printf("Output was:\n\n%s\n", output.c_str());
303    return ok;
304  }
305
306  base::FilePath GetWorkingDir() {
307    std::string home_dir;
308    environment_->GetVar(kHomeEnvName, &home_dir);
309    base::FilePath::StringType native_home_dir(home_dir.begin(),
310                                               home_dir.end());
311    return base::FilePath(native_home_dir).Append(kWorkingDirName);
312  }
313
314  base::FilePath GetReferenceVideosDir() {
315    // FilePath does not tolerate relative paths, and we want to hang the
316    // kReferenceVideosDirName at the same level as Chromium codebase, so we
317    // need to subtract the trailing .../src manually from the path.
318    const size_t src_token_length = 3u;
319    const base::FilePath::StringType src_token(FILE_PATH_LITERAL("src"));
320    base::FilePath::StringType path = GetSourceDir().value();
321    DCHECK_GT(path.size(), src_token_length);
322    std::size_t found = path.rfind(src_token);
323    if (found != std::string::npos)
324      path.replace(found,
325                   src_token_length,
326                   base::FilePath::StringType(FILE_PATH_LITERAL("")));
327    return base::FilePath(path).Append(kReferenceVideosDirName);
328  }
329
330  PeerConnectionServerRunner peerconnection_server_;
331
332 private:
333  base::FilePath GetSourceDir() {
334    base::FilePath source_dir;
335    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
336    return source_dir;
337  }
338
339  base::FilePath GetBrowserDir() {
340    base::FilePath browser_dir;
341    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
342    return browser_dir;
343  }
344
345  base::ProcessHandle pywebsocket_server_;
346  scoped_ptr<base::Environment> environment_;
347  base::FilePath webrtc_reference_video_y4m_;
348};
349
350IN_PROC_BROWSER_TEST_F(WebRtcVideoQualityBrowserTest,
351                       MANUAL_TestVGAVideoQuality) {
352#if defined(OS_WIN)
353  // Fails on XP. http://crbug.com/353078
354  if (base::win::GetVersion() <= base::win::VERSION_XP)
355    return;
356#endif
357
358  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
359      "This is a long-running test; you must specify "
360      "--ui-test-action-max-timeout to have a value of at least 150000.";
361
362  ASSERT_TRUE(HasAllRequiredResources());
363  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
364  ASSERT_TRUE(StartPyWebSocketServer());
365  ASSERT_TRUE(peerconnection_server_.Start());
366
367  content::WebContents* left_tab =
368      OpenPageAndGetUserMediaInNewTabWithConstraints(
369          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
370          kAudioVideoCallConstraints360p);
371  content::WebContents* right_tab =
372      OpenPageAndGetUserMediaInNewTabWithConstraints(
373          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
374          kAudioVideoCallConstraints360p);
375
376  EstablishCall(left_tab, right_tab);
377
378  // Poll slower here to avoid flooding the log with messages: capturing and
379  // sending frames take quite a bit of time.
380  int polling_interval_msec = 1000;
381
382  EXPECT_TRUE(PollingWaitUntil(
383      "doneFrameCapturing()", "done-capturing", right_tab,
384      polling_interval_msec));
385
386  HangUp(left_tab);
387  WaitUntilHangupVerified(left_tab);
388  WaitUntilHangupVerified(right_tab);
389
390  EXPECT_TRUE(PollingWaitUntil(
391      "haveMoreFramesToSend()", "no-more-frames", right_tab,
392      polling_interval_msec));
393
394  // Shut everything down to avoid having the javascript race with the analysis
395  // tools. For instance, dont have console log printouts interleave with the
396  // RESULT lines from the analysis tools (crbug.com/323200).
397  ASSERT_TRUE(peerconnection_server_.Stop());
398  ASSERT_TRUE(ShutdownPyWebSocketServer());
399
400  chrome::CloseWebContents(browser(), left_tab, false);
401  chrome::CloseWebContents(browser(), right_tab, false);
402
403  RunARGBtoI420Converter(
404      k360pWidth, k360pHeight, GetWorkingDir().Append(kCapturedYuvFileName));
405  ASSERT_TRUE(CompareVideosAndPrintResult(
406      k360pWidth,
407      k360pHeight,
408      GetWorkingDir().Append(kCapturedYuvFileName),
409      GetReferenceVideosDir().Append(kReferenceFileName360p).AddExtension(
410          kYuvFileExtension),
411      GetWorkingDir().Append(kStatsFileName)));
412}
413