chrome_webrtc_video_quality_browsertest.cc revision 010d83a9304c5a91596085d917d248abff47903a 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/command_line.h"
6#include "base/environment.h"
7#include "base/file_util.h"
8#include "base/path_service.h"
9#include "base/process/launch.h"
10#include "base/strings/string_split.h"
11#include "base/strings/stringprintf.h"
12#include "base/test/test_timeouts.h"
13#include "base/time/time.h"
14#include "chrome/browser/chrome_notification_types.h"
15#include "chrome/browser/infobars/infobar_service.h"
16#include "chrome/browser/media/media_stream_infobar_delegate.h"
17#include "chrome/browser/media/webrtc_browsertest_base.h"
18#include "chrome/browser/media/webrtc_browsertest_common.h"
19#include "chrome/browser/profiles/profile.h"
20#include "chrome/browser/ui/browser.h"
21#include "chrome/browser/ui/browser_tabstrip.h"
22#include "chrome/browser/ui/tabs/tab_strip_model.h"
23#include "chrome/common/chrome_switches.h"
24#include "chrome/test/base/in_process_browser_test.h"
25#include "chrome/test/base/ui_test_utils.h"
26#include "components/infobars/core/infobar.h"
27#include "content/public/browser/notification_service.h"
28#include "content/public/test/browser_test_utils.h"
29#include "media/base/media_switches.h"
30#include "net/test/embedded_test_server/embedded_test_server.h"
31#include "net/test/python_utils.h"
32#include "testing/perf/perf_test.h"
33#include "ui/gl/gl_switches.h"
34
35// For fine-grained suppression on flaky tests.
36#if defined(OS_WIN)
37#include "base/win/windows_version.h"
38#endif
39
40static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
41#if defined(OS_WIN)
42    FILE_PATH_LITERAL("frame_analyzer.exe");
43#else
44    FILE_PATH_LITERAL("frame_analyzer");
45#endif
46
47static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
48#if defined(OS_WIN)
49    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
50#else
51    FILE_PATH_LITERAL("rgba_to_i420_converter");
52#endif
53
54static const char kHomeEnvName[] =
55#if defined(OS_WIN)
56    "USERPROFILE";
57#else
58    "HOME";
59#endif
60
61// The working dir should be in the user's home folder.
62static const base::FilePath::CharType kWorkingDirName[] =
63    FILE_PATH_LITERAL("webrtc_video_quality");
64static const base::FilePath::CharType kCapturedYuvFileName[] =
65    FILE_PATH_LITERAL("captured_video.yuv");
66static const base::FilePath::CharType kStatsFileName[] =
67    FILE_PATH_LITERAL("stats.txt");
68static const char kMainWebrtcTestHtmlPage[] =
69    "/webrtc/webrtc_jsep01_test.html";
70
71// If you change the port number, don't forget to modify video_extraction.js
72// too!
73static const char kPyWebSocketPortNumber[] = "12221";
74
75static const struct VideoQualityTestConfig {
76  const char* test_name;
77  int width;
78  int height;
79  const char* capture_page;
80  const base::FilePath::CharType* reference_video;
81  const char* constraints;
82} kVideoConfigurations[] = {
83  { "360p", 640, 360,
84    "/webrtc/webrtc_video_quality_test.html",
85    test::kReferenceFileName360p,
86    WebRtcTestBase::kAudioVideoCallConstraints360p },
87  { "720p", 1280, 720,
88    "/webrtc/webrtc_video_quality_test_hd.html",
89    test::kReferenceFileName720p,
90    WebRtcTestBase::kAudioVideoCallConstraints720p },
91};
92
93// Test the video quality of the WebRTC output.
94//
95// Prerequisites: This test case must run on a machine with a chrome playing
96// the video from the reference files located in GetReferenceFilesDir().
97// The file kReferenceY4mFileName.kY4mFileExtension is played using a
98// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
99// comparison.
100//
101// You must also compile the chromium_builder_webrtc target before you run this
102// test to get all the tools built.
103//
104// The external compare_videos.py script also depends on two external
105// executables which must be located in the PATH when running this test.
106// * zxing (see the CPP version at https://code.google.com/p/zxing)
107// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
108//
109// The test runs several custom binaries - rgba_to_i420 converter and
110// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
111// test also runs a stand alone Python implementation of a WebSocket server
112// (pywebsocket) and a barcode_decoder script.
113class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
114    public testing::WithParamInterface<VideoQualityTestConfig> {
115 public:
116  WebRtcVideoQualityBrowserTest()
117      : pywebsocket_server_(0),
118        environment_(base::Environment::Create()) {
119    test_config_ = GetParam();
120  }
121
122  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
123    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
124  }
125
126  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
127    // Set up the command line option with the expected file name. We will check
128    // its existence in HasAllRequiredResources().
129    webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
130        .Append(test_config_.reference_video)
131        .AddExtension(test::kY4mFileExtension);
132    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
133                                   webrtc_reference_video_y4m_);
134    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
135
136    // The video playback will not work without a GPU, so force its use here.
137    command_line->AppendSwitch(switches::kUseGpuInTests);
138  }
139
140  bool HasAllRequiredResources() {
141    if (!base::PathExists(GetWorkingDir())) {
142      LOG(ERROR) << "Cannot find the working directory for the temporary "
143          "files:" << GetWorkingDir().value();
144      return false;
145    }
146
147    // Ensure we have the required input files.
148    return test::HasReferenceFilesInCheckout();
149  }
150
151  bool StartPyWebSocketServer() {
152    base::FilePath path_pywebsocket_dir =
153        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
154    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
155        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
156    base::FilePath path_to_data_handler =
157        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/data/webrtc/wsh"));
158
159    if (!base::PathExists(pywebsocket_server)) {
160      LOG(ERROR) << "Missing pywebsocket server.";
161      return false;
162    }
163    if (!base::PathExists(path_to_data_handler)) {
164      LOG(ERROR) << "Missing data handler for pywebsocket server.";
165      return false;
166    }
167
168    AppendToPythonPath(path_pywebsocket_dir);
169
170    // Note: don't append switches to this command since it will mess up the
171    // -u in the python invocation!
172    CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
173    EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
174
175    pywebsocket_command.AppendArgPath(pywebsocket_server);
176    pywebsocket_command.AppendArg("-p");
177    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
178    pywebsocket_command.AppendArg("-d");
179    pywebsocket_command.AppendArgPath(path_to_data_handler);
180
181    VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
182    return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
183                               &pywebsocket_server_);
184  }
185
186  bool ShutdownPyWebSocketServer() {
187    return base::KillProcess(pywebsocket_server_, 0, false);
188  }
189
190  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
191  // which should contain frames of size |width| x |height|.
192  //
193  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
194  // should be build prior to running this test. The resulting binary should
195  // live next to Chrome.
196  bool RunARGBtoI420Converter(int width,
197                              int height,
198                              const base::FilePath& captured_video_filename) {
199    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
200        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
201
202    if (!base::PathExists(path_to_converter)) {
203      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
204          << path_to_converter.value();
205      return false;
206    }
207
208    CommandLine converter_command(path_to_converter);
209    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
210    converter_command.AppendSwitchPath("--output_file",
211                                       captured_video_filename);
212    converter_command.AppendSwitchASCII("--width",
213                                        base::StringPrintf("%d", width));
214    converter_command.AppendSwitchASCII("--height",
215                                        base::StringPrintf("%d", height));
216    converter_command.AppendSwitchASCII("--delete_frames", "true");
217
218    // We produce an output file that will later be used as an input to the
219    // barcode decoder and frame analyzer tools.
220    VLOG(0) << "Running " << converter_command.GetCommandLineString();
221    std::string result;
222    bool ok = base::GetAppOutput(converter_command, &result);
223    VLOG(0) << "Output was:\n\n" << result;
224    return ok;
225  }
226
227  // Compares the |captured_video_filename| with the |reference_video_filename|.
228  //
229  // The barcode decoder decodes the captured video containing barcodes overlaid
230  // into every frame of the video (produced by rgba_to_i420_converter). It
231  // produces a set of PNG images and a |stats_file| that maps each captured
232  // frame to a frame in the reference video. The frames should be of size
233  // |width| x |height|.
234  // All measurements calculated are printed as perf parsable numbers to stdout.
235  bool CompareVideosAndPrintResult(
236      const char* test_label,
237      int width,
238      int height,
239      const base::FilePath& captured_video_filename,
240      const base::FilePath& reference_video_filename,
241      const base::FilePath& stats_file) {
242
243    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
244        GetBrowserDir().Append(kFrameAnalyzerExecutable));
245    base::FilePath path_to_compare_script = GetSourceDir().Append(
246        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
247
248    if (!base::PathExists(path_to_analyzer)) {
249      LOG(ERROR) << "Missing frame analyzer: should be in "
250          << path_to_analyzer.value();
251      return false;
252    }
253    if (!base::PathExists(path_to_compare_script)) {
254      LOG(ERROR) << "Missing video compare script: should be in "
255          << path_to_compare_script.value();
256      return false;
257    }
258
259    // Note: don't append switches to this command since it will mess up the
260    // -u in the python invocation!
261    CommandLine compare_command(CommandLine::NO_PROGRAM);
262    EXPECT_TRUE(GetPythonCommand(&compare_command));
263
264    compare_command.AppendArgPath(path_to_compare_script);
265    compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
266    compare_command.AppendArg("--ref_video");
267    compare_command.AppendArgPath(reference_video_filename);
268    compare_command.AppendArg("--test_video");
269    compare_command.AppendArgPath(captured_video_filename);
270    compare_command.AppendArg("--frame_analyzer");
271    compare_command.AppendArgPath(path_to_analyzer);
272    compare_command.AppendArg("--yuv_frame_width");
273    compare_command.AppendArg(base::StringPrintf("%d", width));
274    compare_command.AppendArg("--yuv_frame_height");
275    compare_command.AppendArg(base::StringPrintf("%d", height));
276    compare_command.AppendArg("--stats_file");
277    compare_command.AppendArgPath(stats_file);
278
279    VLOG(0) << "Running " << compare_command.GetCommandLineString();
280    std::string output;
281    bool ok = base::GetAppOutput(compare_command, &output);
282    // Print to stdout to ensure the perf numbers are parsed properly by the
283    // buildbot step.
284    printf("Output was:\n\n%s\n", output.c_str());
285    return ok;
286  }
287
288  base::FilePath GetWorkingDir() {
289    std::string home_dir;
290    environment_->GetVar(kHomeEnvName, &home_dir);
291    base::FilePath::StringType native_home_dir(home_dir.begin(),
292                                               home_dir.end());
293    return base::FilePath(native_home_dir).Append(kWorkingDirName);
294  }
295
296 protected:
297  VideoQualityTestConfig test_config_;
298
299 private:
300  base::FilePath GetSourceDir() {
301    base::FilePath source_dir;
302    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
303    return source_dir;
304  }
305
306  base::FilePath GetBrowserDir() {
307    base::FilePath browser_dir;
308    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
309    return browser_dir;
310  }
311
312  base::ProcessHandle pywebsocket_server_;
313  scoped_ptr<base::Environment> environment_;
314  base::FilePath webrtc_reference_video_y4m_;
315};
316
317INSTANTIATE_TEST_CASE_P(
318    WebRtcVideoQualityBrowserTests,
319    WebRtcVideoQualityBrowserTest,
320    testing::ValuesIn(kVideoConfigurations));
321
322IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
323                       MANUAL_TestVideoQuality) {
324
325#if defined(OS_WIN)
326  // Fails on XP. http://crbug.com/353078
327  if (base::win::GetVersion() <= base::win::VERSION_XP)
328    return;
329#endif
330
331  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
332      "This is a long-running test; you must specify "
333      "--ui-test-action-max-timeout to have a value of at least 150000.";
334  ASSERT_TRUE(HasAllRequiredResources());
335  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
336  ASSERT_TRUE(StartPyWebSocketServer());
337
338  content::WebContents* left_tab =
339      OpenPageAndGetUserMediaInNewTabWithConstraints(
340          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
341          test_config_.constraints);
342  content::WebContents* right_tab =
343      OpenPageAndGetUserMediaInNewTabWithConstraints(
344          embedded_test_server()->GetURL(test_config_.capture_page),
345          test_config_.constraints);
346
347  SetupPeerconnectionWithLocalStream(left_tab);
348  SetupPeerconnectionWithLocalStream(right_tab);
349
350  NegotiateCall(left_tab, right_tab);
351
352  // Poll slower here to avoid flooding the log with messages: capturing and
353  // sending frames take quite a bit of time.
354  int polling_interval_msec = 1000;
355
356  EXPECT_TRUE(test::PollingWaitUntil(
357      "doneFrameCapturing()", "done-capturing", right_tab,
358      polling_interval_msec));
359
360  HangUp(left_tab);
361
362  EXPECT_TRUE(test::PollingWaitUntil(
363      "haveMoreFramesToSend()", "no-more-frames", right_tab,
364      polling_interval_msec));
365
366  // Shut everything down to avoid having the javascript race with the analysis
367  // tools. For instance, dont have console log printouts interleave with the
368  // RESULT lines from the analysis tools (crbug.com/323200).
369  ASSERT_TRUE(ShutdownPyWebSocketServer());
370
371  chrome::CloseWebContents(browser(), left_tab, false);
372  chrome::CloseWebContents(browser(), right_tab, false);
373
374  RunARGBtoI420Converter(
375      test_config_.width, test_config_.height,
376      GetWorkingDir().Append(kCapturedYuvFileName));
377  ASSERT_TRUE(CompareVideosAndPrintResult(
378      test_config_.test_name,
379      test_config_.width,
380      test_config_.height,
381      GetWorkingDir().Append(kCapturedYuvFileName),
382      test::GetReferenceFilesDir()
383          .Append(test_config_.reference_video)
384          .AddExtension(test::kYuvFileExtension),
385      GetWorkingDir().Append(kStatsFileName)));
386}
387