1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/base64.h"
6#include "base/command_line.h"
7#include "base/environment.h"
8#include "base/files/file.h"
9#include "base/files/file_util.h"
10#include "base/files/scoped_temp_dir.h"
11#include "base/path_service.h"
12#include "base/process/launch.h"
13#include "base/strings/string_number_conversions.h"
14#include "base/strings/string_split.h"
15#include "base/strings/stringprintf.h"
16#include "base/test/test_timeouts.h"
17#include "base/time/time.h"
18#include "chrome/browser/chrome_notification_types.h"
19#include "chrome/browser/infobars/infobar_service.h"
20#include "chrome/browser/media/media_stream_infobar_delegate.h"
21#include "chrome/browser/media/webrtc_browsertest_base.h"
22#include "chrome/browser/media/webrtc_browsertest_common.h"
23#include "chrome/browser/profiles/profile.h"
24#include "chrome/browser/ui/browser.h"
25#include "chrome/browser/ui/browser_tabstrip.h"
26#include "chrome/browser/ui/tabs/tab_strip_model.h"
27#include "chrome/common/chrome_switches.h"
28#include "chrome/test/base/in_process_browser_test.h"
29#include "chrome/test/base/ui_test_utils.h"
30#include "components/infobars/core/infobar.h"
31#include "content/public/browser/notification_service.h"
32#include "content/public/test/browser_test_utils.h"
33#include "media/base/media_switches.h"
34#include "net/test/embedded_test_server/embedded_test_server.h"
35#include "net/test/python_utils.h"
36#include "testing/perf/perf_test.h"
37#include "ui/gl/gl_switches.h"
38
39static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
40#if defined(OS_WIN)
41    FILE_PATH_LITERAL("frame_analyzer.exe");
42#else
43    FILE_PATH_LITERAL("frame_analyzer");
44#endif
45
46static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
47#if defined(OS_WIN)
48    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
49#else
50    FILE_PATH_LITERAL("rgba_to_i420_converter");
51#endif
52
53static const base::FilePath::CharType kCapturedYuvFileName[] =
54    FILE_PATH_LITERAL("captured_video.yuv");
55static const base::FilePath::CharType kStatsFileName[] =
56    FILE_PATH_LITERAL("stats.txt");
57static const char kMainWebrtcTestHtmlPage[] =
58    "/webrtc/webrtc_jsep01_test.html";
59static const char kCapturingWebrtcHtmlPage[] =
60    "/webrtc/webrtc_video_quality_test.html";
61
62static const struct VideoQualityTestConfig {
63  const char* test_name;
64  int width;
65  int height;
66  const base::FilePath::CharType* reference_video;
67  const char* constraints;
68} kVideoConfigurations[] = {
69  { "360p", 640, 360,
70    test::kReferenceFileName360p,
71    WebRtcTestBase::kAudioVideoCallConstraints360p },
72  { "720p", 1280, 720,
73    test::kReferenceFileName720p,
74    WebRtcTestBase::kAudioVideoCallConstraints720p },
75};
76
77// Test the video quality of the WebRTC output.
78//
79// Prerequisites: This test case must run on a machine with a chrome playing
80// the video from the reference files located in GetReferenceFilesDir().
81// The file kReferenceY4mFileName.kY4mFileExtension is played using a
82// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
83// comparison.
84//
85// You must also compile the chromium_builder_webrtc target before you run this
86// test to get all the tools built.
87//
88// The external compare_videos.py script also depends on two external
89// executables which must be located in the PATH when running this test.
90// * zxing (see the CPP version at https://code.google.com/p/zxing)
91// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
92//
93// The test runs several custom binaries - rgba_to_i420 converter and
94// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
95// test also runs a stand alone Python implementation of a WebSocket server
96// (pywebsocket) and a barcode_decoder script.
97class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
98    public testing::WithParamInterface<VideoQualityTestConfig> {
99 public:
100  WebRtcVideoQualityBrowserTest()
101      : environment_(base::Environment::Create()) {
102    test_config_ = GetParam();
103  }
104
105  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
106    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
107
108    ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir());
109  }
110
111  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
112    // Set up the command line option with the expected file name. We will check
113    // its existence in HasAllRequiredResources().
114    webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
115        .Append(test_config_.reference_video)
116        .AddExtension(test::kY4mFileExtension);
117    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
118                                   webrtc_reference_video_y4m_);
119    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
120
121    // The video playback will not work without a GPU, so force its use here.
122    command_line->AppendSwitch(switches::kUseGpuInTests);
123  }
124
125  // Writes all frames we've captured so far by grabbing them from the
126  // javascript and writing them to the temporary work directory.
127  void WriteCapturedFramesToWorkingDir(content::WebContents* capturing_tab) {
128    int num_frames = 0;
129    std::string response =
130        ExecuteJavascript("getTotalNumberCapturedFrames()", capturing_tab);
131    ASSERT_TRUE(base::StringToInt(response, &num_frames)) <<
132        "Failed to retrieve frame count: got " << response;
133    ASSERT_NE(0, num_frames) << "Failed to capture any frames.";
134
135    for (int i = 0; i < num_frames; i++) {
136      std::string base64_encoded_frame =
137          ExecuteJavascript(base::StringPrintf("getOneCapturedFrame(%d)", i),
138                            capturing_tab);
139      std::string decoded_frame;
140      ASSERT_TRUE(base::Base64Decode(base64_encoded_frame, &decoded_frame))
141          << "Failed to decode frame data '" << base64_encoded_frame << "'.";
142
143      std::string file_name = base::StringPrintf("frame_%04d", i);
144      base::File frame_file(GetWorkingDir().AppendASCII(file_name),
145                            base::File::FLAG_CREATE | base::File::FLAG_WRITE);
146      size_t written = frame_file.Write(0, decoded_frame.c_str(),
147                                        decoded_frame.length());
148      ASSERT_EQ(decoded_frame.length(), written);
149    }
150  }
151
152  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
153  // which should contain frames of size |width| x |height|.
154  //
155  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
156  // should be build prior to running this test. The resulting binary should
157  // live next to Chrome.
158  bool RunARGBtoI420Converter(int width,
159                              int height,
160                              const base::FilePath& captured_video_filename) {
161    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
162        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
163
164    if (!base::PathExists(path_to_converter)) {
165      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
166          << path_to_converter.value();
167      return false;
168    }
169
170    CommandLine converter_command(path_to_converter);
171    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
172    converter_command.AppendSwitchPath("--output_file",
173                                       captured_video_filename);
174    converter_command.AppendSwitchASCII("--width",
175                                        base::StringPrintf("%d", width));
176    converter_command.AppendSwitchASCII("--height",
177                                        base::StringPrintf("%d", height));
178    converter_command.AppendSwitchASCII("--delete_frames", "true");
179
180    // We produce an output file that will later be used as an input to the
181    // barcode decoder and frame analyzer tools.
182    VLOG(0) << "Running " << converter_command.GetCommandLineString();
183    std::string result;
184    bool ok = base::GetAppOutput(converter_command, &result);
185    VLOG(0) << "Output was:\n\n" << result;
186    return ok;
187  }
188
189  // Compares the |captured_video_filename| with the |reference_video_filename|.
190  //
191  // The barcode decoder decodes the captured video containing barcodes overlaid
192  // into every frame of the video (produced by rgba_to_i420_converter). It
193  // produces a set of PNG images and a |stats_file| that maps each captured
194  // frame to a frame in the reference video. The frames should be of size
195  // |width| x |height|.
196  // All measurements calculated are printed as perf parsable numbers to stdout.
197  bool CompareVideosAndPrintResult(
198      const char* test_label,
199      int width,
200      int height,
201      const base::FilePath& captured_video_filename,
202      const base::FilePath& reference_video_filename,
203      const base::FilePath& stats_file) {
204
205    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
206        GetBrowserDir().Append(kFrameAnalyzerExecutable));
207    base::FilePath path_to_compare_script = GetSourceDir().Append(
208        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
209
210    if (!base::PathExists(path_to_analyzer)) {
211      LOG(ERROR) << "Missing frame analyzer: should be in "
212          << path_to_analyzer.value();
213      return false;
214    }
215    if (!base::PathExists(path_to_compare_script)) {
216      LOG(ERROR) << "Missing video compare script: should be in "
217          << path_to_compare_script.value();
218      return false;
219    }
220
221    // Note: don't append switches to this command since it will mess up the
222    // -u in the python invocation!
223    CommandLine compare_command(CommandLine::NO_PROGRAM);
224    EXPECT_TRUE(GetPythonCommand(&compare_command));
225
226    compare_command.AppendArgPath(path_to_compare_script);
227    compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
228    compare_command.AppendArg("--ref_video");
229    compare_command.AppendArgPath(reference_video_filename);
230    compare_command.AppendArg("--test_video");
231    compare_command.AppendArgPath(captured_video_filename);
232    compare_command.AppendArg("--frame_analyzer");
233    compare_command.AppendArgPath(path_to_analyzer);
234    compare_command.AppendArg("--yuv_frame_width");
235    compare_command.AppendArg(base::StringPrintf("%d", width));
236    compare_command.AppendArg("--yuv_frame_height");
237    compare_command.AppendArg(base::StringPrintf("%d", height));
238    compare_command.AppendArg("--stats_file");
239    compare_command.AppendArgPath(stats_file);
240
241    VLOG(0) << "Running " << compare_command.GetCommandLineString();
242    std::string output;
243    bool ok = base::GetAppOutput(compare_command, &output);
244    // Print to stdout to ensure the perf numbers are parsed properly by the
245    // buildbot step.
246    printf("Output was:\n\n%s\n", output.c_str());
247    return ok;
248  }
249
250 protected:
251  VideoQualityTestConfig test_config_;
252
253  base::FilePath GetWorkingDir() {
254    return temp_working_dir_.path();
255  }
256
257 private:
258  base::FilePath GetSourceDir() {
259    base::FilePath source_dir;
260    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
261    return source_dir;
262  }
263
264  base::FilePath GetBrowserDir() {
265    base::FilePath browser_dir;
266    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
267    return browser_dir;
268  }
269
270  scoped_ptr<base::Environment> environment_;
271  base::FilePath webrtc_reference_video_y4m_;
272  base::ScopedTempDir temp_working_dir_;
273};
274
275INSTANTIATE_TEST_CASE_P(
276    WebRtcVideoQualityBrowserTests,
277    WebRtcVideoQualityBrowserTest,
278    testing::ValuesIn(kVideoConfigurations));
279
280IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
281                       MANUAL_TestVideoQuality) {
282  if (OnWinXp())
283    return;  // Fails on XP. http://crbug.com/353078.
284
285  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
286      "This is a long-running test; you must specify "
287      "--ui-test-action-max-timeout to have a value of at least 150000.";
288  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
289  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
290
291  content::WebContents* left_tab =
292      OpenPageAndGetUserMediaInNewTabWithConstraints(
293          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
294          test_config_.constraints);
295  content::WebContents* right_tab =
296      OpenPageAndGetUserMediaInNewTabWithConstraints(
297          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
298          test_config_.constraints);
299
300  SetupPeerconnectionWithLocalStream(left_tab);
301  SetupPeerconnectionWithLocalStream(right_tab);
302
303  NegotiateCall(left_tab, right_tab);
304
305  // Poll slower here to avoid flooding the log with messages: capturing and
306  // sending frames take quite a bit of time.
307  int polling_interval_msec = 1000;
308
309  EXPECT_TRUE(test::PollingWaitUntil(
310      "doneFrameCapturing()", "done-capturing", right_tab,
311      polling_interval_msec));
312
313  HangUp(left_tab);
314
315  WriteCapturedFramesToWorkingDir(right_tab);
316
317  // Shut everything down to avoid having the javascript race with the analysis
318  // tools. For instance, dont have console log printouts interleave with the
319  // RESULT lines from the analysis tools (crbug.com/323200).
320  chrome::CloseWebContents(browser(), left_tab, false);
321  chrome::CloseWebContents(browser(), right_tab, false);
322
323  ASSERT_TRUE(RunARGBtoI420Converter(
324      test_config_.width, test_config_.height,
325      GetWorkingDir().Append(kCapturedYuvFileName)));
326  ASSERT_TRUE(CompareVideosAndPrintResult(
327      test_config_.test_name,
328      test_config_.width,
329      test_config_.height,
330      GetWorkingDir().Append(kCapturedYuvFileName),
331      test::GetReferenceFilesDir()
332          .Append(test_config_.reference_video)
333          .AddExtension(test::kYuvFileExtension),
334      GetWorkingDir().Append(kStatsFileName)));
335}
336