chrome_webrtc_video_quality_browsertest.cc revision cedac228d2dd51db4b79ea1e72c7f249408ee061 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/command_line.h"
6#include "base/environment.h"
7#include "base/file_util.h"
8#include "base/path_service.h"
9#include "base/process/launch.h"
10#include "base/strings/string_split.h"
11#include "base/strings/stringprintf.h"
12#include "base/test/test_timeouts.h"
13#include "base/time/time.h"
14#include "chrome/browser/chrome_notification_types.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "components/infobars/core/infobar.h"
27#include "content/public/browser/notification_service.h"
28#include "content/public/test/browser_test_utils.h"
29#include "media/base/media_switches.h"
30#include "net/test/embedded_test_server/embedded_test_server.h"
31#include "net/test/python_utils.h"
32#include "testing/perf/perf_test.h"
33#include "ui/gl/gl_switches.h"
34
35static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
36#if defined(OS_WIN)
37    FILE_PATH_LITERAL("frame_analyzer.exe");
38#else
39    FILE_PATH_LITERAL("frame_analyzer");
40#endif
41
42static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
43#if defined(OS_WIN)
44    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
45#else
46    FILE_PATH_LITERAL("rgba_to_i420_converter");
47#endif
48
49static const char kHomeEnvName[] =
50#if defined(OS_WIN)
51    "USERPROFILE";
52#else
53    "HOME";
54#endif
55
56// The working dir should be in the user's home folder.
57static const base::FilePath::CharType kWorkingDirName[] =
58    FILE_PATH_LITERAL("webrtc_video_quality");
59static const base::FilePath::CharType kCapturedYuvFileName[] =
60    FILE_PATH_LITERAL("captured_video.yuv");
61static const base::FilePath::CharType kStatsFileName[] =
62    FILE_PATH_LITERAL("stats.txt");
63static const char kMainWebrtcTestHtmlPage[] =
64    "/webrtc/webrtc_jsep01_test.html";
65static const char kCapturingWebrtcHtmlPage[] =
66    "/webrtc/webrtc_video_quality_test.html";
67
68// If you change the port number, don't forget to modify video_extraction.js
69// too!
70static const char kPyWebSocketPortNumber[] = "12221";
71
72static const struct VideoQualityTestConfig {
73  const char* test_name;
74  int width;
75  int height;
76  const base::FilePath::CharType* reference_video;
77  const char* constraints;
78} kVideoConfigurations[] = {
79  { "360p", 640, 360,
80    test::kReferenceFileName360p,
81    WebRtcTestBase::kAudioVideoCallConstraints360p },
82  { "720p", 1280, 720,
83    test::kReferenceFileName720p,
84    WebRtcTestBase::kAudioVideoCallConstraints720p },
85};
86
87// Test the video quality of the WebRTC output.
88//
89// Prerequisites: This test case must run on a machine with a chrome playing
90// the video from the reference files located in GetReferenceFilesDir().
91// The file kReferenceY4mFileName.kY4mFileExtension is played using a
92// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
93// comparison.
94//
95// You must also compile the chromium_builder_webrtc target before you run this
96// test to get all the tools built.
97//
98// The external compare_videos.py script also depends on two external
99// executables which must be located in the PATH when running this test.
100// * zxing (see the CPP version at https://code.google.com/p/zxing)
101// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
102//
103// The test runs several custom binaries - rgba_to_i420 converter and
104// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
105// test also runs a stand alone Python implementation of a WebSocket server
106// (pywebsocket) and a barcode_decoder script.
107class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
108    public testing::WithParamInterface<VideoQualityTestConfig> {
109 public:
110  WebRtcVideoQualityBrowserTest()
111      : pywebsocket_server_(0),
112        environment_(base::Environment::Create()) {
113    test_config_ = GetParam();
114  }
115
116  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
117    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
118  }
119
120  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
121    // Set up the command line option with the expected file name. We will check
122    // its existence in HasAllRequiredResources().
123    webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
124        .Append(test_config_.reference_video)
125        .AddExtension(test::kY4mFileExtension);
126    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
127                                   webrtc_reference_video_y4m_);
128    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
129
130    // The video playback will not work without a GPU, so force its use here.
131    command_line->AppendSwitch(switches::kUseGpuInTests);
132  }
133
134  bool HasAllRequiredResources() {
135    if (!base::PathExists(GetWorkingDir())) {
136      LOG(ERROR) << "Cannot find the working directory for the temporary "
137          "files:" << GetWorkingDir().value();
138      return false;
139    }
140
141    // Ensure we have the required input files.
142    return test::HasReferenceFilesInCheckout();
143  }
144
145  bool StartPyWebSocketServer() {
146    base::FilePath path_pywebsocket_dir =
147        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
148    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
149        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
150    base::FilePath path_to_data_handler =
151        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/data/webrtc/wsh"));
152
153    if (!base::PathExists(pywebsocket_server)) {
154      LOG(ERROR) << "Missing pywebsocket server.";
155      return false;
156    }
157    if (!base::PathExists(path_to_data_handler)) {
158      LOG(ERROR) << "Missing data handler for pywebsocket server.";
159      return false;
160    }
161
162    AppendToPythonPath(path_pywebsocket_dir);
163
164    // Note: don't append switches to this command since it will mess up the
165    // -u in the python invocation!
166    CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
167    EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
168
169    pywebsocket_command.AppendArgPath(pywebsocket_server);
170    pywebsocket_command.AppendArg("-p");
171    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
172    pywebsocket_command.AppendArg("-d");
173    pywebsocket_command.AppendArgPath(path_to_data_handler);
174
175    VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
176    return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
177                               &pywebsocket_server_);
178  }
179
180  bool ShutdownPyWebSocketServer() {
181    return base::KillProcess(pywebsocket_server_, 0, false);
182  }
183
184  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
185  // which should contain frames of size |width| x |height|.
186  //
187  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
188  // should be build prior to running this test. The resulting binary should
189  // live next to Chrome.
190  bool RunARGBtoI420Converter(int width,
191                              int height,
192                              const base::FilePath& captured_video_filename) {
193    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
194        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
195
196    if (!base::PathExists(path_to_converter)) {
197      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
198          << path_to_converter.value();
199      return false;
200    }
201
202    CommandLine converter_command(path_to_converter);
203    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
204    converter_command.AppendSwitchPath("--output_file",
205                                       captured_video_filename);
206    converter_command.AppendSwitchASCII("--width",
207                                        base::StringPrintf("%d", width));
208    converter_command.AppendSwitchASCII("--height",
209                                        base::StringPrintf("%d", height));
210    converter_command.AppendSwitchASCII("--delete_frames", "true");
211
212    // We produce an output file that will later be used as an input to the
213    // barcode decoder and frame analyzer tools.
214    VLOG(0) << "Running " << converter_command.GetCommandLineString();
215    std::string result;
216    bool ok = base::GetAppOutput(converter_command, &result);
217    VLOG(0) << "Output was:\n\n" << result;
218    return ok;
219  }
220
221  // Compares the |captured_video_filename| with the |reference_video_filename|.
222  //
223  // The barcode decoder decodes the captured video containing barcodes overlaid
224  // into every frame of the video (produced by rgba_to_i420_converter). It
225  // produces a set of PNG images and a |stats_file| that maps each captured
226  // frame to a frame in the reference video. The frames should be of size
227  // |width| x |height|.
228  // All measurements calculated are printed as perf parsable numbers to stdout.
229  bool CompareVideosAndPrintResult(
230      const char* test_label,
231      int width,
232      int height,
233      const base::FilePath& captured_video_filename,
234      const base::FilePath& reference_video_filename,
235      const base::FilePath& stats_file) {
236
237    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
238        GetBrowserDir().Append(kFrameAnalyzerExecutable));
239    base::FilePath path_to_compare_script = GetSourceDir().Append(
240        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
241
242    if (!base::PathExists(path_to_analyzer)) {
243      LOG(ERROR) << "Missing frame analyzer: should be in "
244          << path_to_analyzer.value();
245      return false;
246    }
247    if (!base::PathExists(path_to_compare_script)) {
248      LOG(ERROR) << "Missing video compare script: should be in "
249          << path_to_compare_script.value();
250      return false;
251    }
252
253    // Note: don't append switches to this command since it will mess up the
254    // -u in the python invocation!
255    CommandLine compare_command(CommandLine::NO_PROGRAM);
256    EXPECT_TRUE(GetPythonCommand(&compare_command));
257
258    compare_command.AppendArgPath(path_to_compare_script);
259    compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
260    compare_command.AppendArg("--ref_video");
261    compare_command.AppendArgPath(reference_video_filename);
262    compare_command.AppendArg("--test_video");
263    compare_command.AppendArgPath(captured_video_filename);
264    compare_command.AppendArg("--frame_analyzer");
265    compare_command.AppendArgPath(path_to_analyzer);
266    compare_command.AppendArg("--yuv_frame_width");
267    compare_command.AppendArg(base::StringPrintf("%d", width));
268    compare_command.AppendArg("--yuv_frame_height");
269    compare_command.AppendArg(base::StringPrintf("%d", height));
270    compare_command.AppendArg("--stats_file");
271    compare_command.AppendArgPath(stats_file);
272
273    VLOG(0) << "Running " << compare_command.GetCommandLineString();
274    std::string output;
275    bool ok = base::GetAppOutput(compare_command, &output);
276    // Print to stdout to ensure the perf numbers are parsed properly by the
277    // buildbot step.
278    printf("Output was:\n\n%s\n", output.c_str());
279    return ok;
280  }
281
282  base::FilePath GetWorkingDir() {
283    std::string home_dir;
284    environment_->GetVar(kHomeEnvName, &home_dir);
285    base::FilePath::StringType native_home_dir(home_dir.begin(),
286                                               home_dir.end());
287    return base::FilePath(native_home_dir).Append(kWorkingDirName);
288  }
289
290 protected:
291  VideoQualityTestConfig test_config_;
292
293 private:
294  base::FilePath GetSourceDir() {
295    base::FilePath source_dir;
296    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
297    return source_dir;
298  }
299
300  base::FilePath GetBrowserDir() {
301    base::FilePath browser_dir;
302    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
303    return browser_dir;
304  }
305
306  base::ProcessHandle pywebsocket_server_;
307  scoped_ptr<base::Environment> environment_;
308  base::FilePath webrtc_reference_video_y4m_;
309};
310
311INSTANTIATE_TEST_CASE_P(
312    WebRtcVideoQualityBrowserTests,
313    WebRtcVideoQualityBrowserTest,
314    testing::ValuesIn(kVideoConfigurations));
315
316IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
317                       MANUAL_TestVideoQuality) {
318  if (OnWinXp())
319    return;  // Fails on XP. http://crbug.com/353078.
320
321  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
322      "This is a long-running test; you must specify "
323      "--ui-test-action-max-timeout to have a value of at least 150000.";
324  ASSERT_TRUE(HasAllRequiredResources());
325  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
326  ASSERT_TRUE(StartPyWebSocketServer());
327
328  content::WebContents* left_tab =
329      OpenPageAndGetUserMediaInNewTabWithConstraints(
330          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
331          test_config_.constraints);
332  content::WebContents* right_tab =
333      OpenPageAndGetUserMediaInNewTabWithConstraints(
334          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
335          test_config_.constraints);
336
337  SetupPeerconnectionWithLocalStream(left_tab);
338  SetupPeerconnectionWithLocalStream(right_tab);
339
340  NegotiateCall(left_tab, right_tab);
341
342  // Poll slower here to avoid flooding the log with messages: capturing and
343  // sending frames take quite a bit of time.
344  int polling_interval_msec = 1000;
345
346  EXPECT_TRUE(test::PollingWaitUntil(
347      "doneFrameCapturing()", "done-capturing", right_tab,
348      polling_interval_msec));
349
350  HangUp(left_tab);
351
352  EXPECT_TRUE(test::PollingWaitUntil(
353      "haveMoreFramesToSend()", "no-more-frames", right_tab,
354      polling_interval_msec));
355
356  // Shut everything down to avoid having the javascript race with the analysis
357  // tools. For instance, dont have console log printouts interleave with the
358  // RESULT lines from the analysis tools (crbug.com/323200).
359  ASSERT_TRUE(ShutdownPyWebSocketServer());
360
361  chrome::CloseWebContents(browser(), left_tab, false);
362  chrome::CloseWebContents(browser(), right_tab, false);
363
364  RunARGBtoI420Converter(
365      test_config_.width, test_config_.height,
366      GetWorkingDir().Append(kCapturedYuvFileName));
367  ASSERT_TRUE(CompareVideosAndPrintResult(
368      test_config_.test_name,
369      test_config_.width,
370      test_config_.height,
371      GetWorkingDir().Append(kCapturedYuvFileName),
372      test::GetReferenceFilesDir()
373          .Append(test_config_.reference_video)
374          .AddExtension(test::kYuvFileExtension),
375      GetWorkingDir().Append(kStatsFileName)));
376}
377