1eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// Copyright 2013 The Chromium Authors. All rights reserved.
2eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// Use of this source code is governed by a BSD-style license that can be
3eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// found in the LICENSE file.
4eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
5f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#include "base/base64.h"
6c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch#include "base/command_line.h"
7eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/environment.h"
8f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#include "base/files/file.h"
91320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci#include "base/files/file_util.h"
10f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#include "base/files/scoped_temp_dir.h"
11eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/path_service.h"
12bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch#include "base/process/launch.h"
13f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)#include "base/strings/string_number_conversions.h"
14eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/strings/string_split.h"
15eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/strings/stringprintf.h"
16eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/test/test_timeouts.h"
17eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "base/time/time.h"
187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch#include "chrome/browser/chrome_notification_types.h"
19eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/infobars/infobar_service.h"
20eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/media/media_stream_infobar_delegate.h"
21558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch#include "chrome/browser/media/webrtc_browsertest_base.h"
22a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch#include "chrome/browser/media/webrtc_browsertest_common.h"
23eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/profiles/profile.h"
24eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/ui/browser.h"
25eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/ui/browser_tabstrip.h"
26eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/browser/ui/tabs/tab_strip_model.h"
27eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/common/chrome_switches.h"
28eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/test/base/in_process_browser_test.h"
29eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "chrome/test/base/ui_test_utils.h"
300529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch#include "components/infobars/core/infobar.h"
31eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "content/public/browser/notification_service.h"
32eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "content/public/test/browser_test_utils.h"
335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "media/base/media_switches.h"
34424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "net/test/embedded_test_server/embedded_test_server.h"
35eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include "net/test/python_utils.h"
3658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "testing/perf/perf_test.h"
375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "ui/gl/gl_switches.h"
38a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)
39eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic const base::FilePath::CharType kFrameAnalyzerExecutable[] =
40eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#if defined(OS_WIN)
41eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("frame_analyzer.exe");
42eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#else
43eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("frame_analyzer");
44eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#endif
45eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
46eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
47eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#if defined(OS_WIN)
48eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
49eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#else
50eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("rgba_to_i420_converter");
51eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#endif
52eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
53eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic const base::FilePath::CharType kCapturedYuvFileName[] =
54eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("captured_video.yuv");
55eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic const base::FilePath::CharType kStatsFileName[] =
56eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    FILE_PATH_LITERAL("stats.txt");
57eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochstatic const char kMainWebrtcTestHtmlPage[] =
58424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    "/webrtc/webrtc_jsep01_test.html";
59cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)static const char kCapturingWebrtcHtmlPage[] =
60cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    "/webrtc/webrtc_video_quality_test.html";
61eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
625c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liustatic const struct VideoQualityTestConfig {
635c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  const char* test_name;
645c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  int width;
655c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  int height;
665c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  const base::FilePath::CharType* reference_video;
675c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  const char* constraints;
685c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu} kVideoConfigurations[] = {
695c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  { "360p", 640, 360,
705c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    test::kReferenceFileName360p,
715c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    WebRtcTestBase::kAudioVideoCallConstraints360p },
725c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  { "720p", 1280, 720,
735c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    test::kReferenceFileName720p,
745c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    WebRtcTestBase::kAudioVideoCallConstraints720p },
755c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu};
765c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu
77eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// Test the video quality of the WebRTC output.
78eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch//
79a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// Prerequisites: This test case must run on a machine with a chrome playing
80c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch// the video from the reference files located in GetReferenceFilesDir().
81a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// The file kReferenceY4mFileName.kY4mFileExtension is played using a
82a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
83a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)// comparison.
84eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch//
85eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// You must also compile the chromium_builder_webrtc target before you run this
86eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// test to get all the tools built.
87eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch//
88eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// The external compare_videos.py script also depends on two external
89eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// executables which must be located in the PATH when running this test.
90eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// * zxing (see the CPP version at https://code.google.com/p/zxing)
91eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
92eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch//
93010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)// The test runs several custom binaries - rgba_to_i420 converter and
94eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
95eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// test also runs a stand alone Python implementation of a WebSocket server
96eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch// (pywebsocket) and a barcode_decoder script.
975c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuclass WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
985c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    public testing::WithParamInterface<VideoQualityTestConfig> {
99eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch public:
1005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)  WebRtcVideoQualityBrowserTest()
101f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      : environment_(base::Environment::Create()) {
1025c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    test_config_ = GetParam();
1035c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  }
104eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
1052385ea399aae016c0806a4f9ef3c9cfe3d2a39dfBen Murdoch  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
106a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
107f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
108f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir());
109eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
110eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
111eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
112a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    // Set up the command line option with the expected file name. We will check
113a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    // its existence in HasAllRequiredResources().
114c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch    webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
1155c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu        .Append(test_config_.reference_video)
116e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch        .AddExtension(test::kY4mFileExtension);
117a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
118a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)                                   webrtc_reference_video_y4m_);
119a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
1205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)
1213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    // The video playback will not work without a GPU, so force its use here.
1223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    command_line->AppendSwitch(switches::kUseGpuInTests);
123eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
124eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
125f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  // Writes all frames we've captured so far by grabbing them from the
126f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  // javascript and writing them to the temporary work directory.
127f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  void WriteCapturedFramesToWorkingDir(content::WebContents* capturing_tab) {
128f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    int num_frames = 0;
129f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    std::string response =
130f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)        ExecuteJavascript("getTotalNumberCapturedFrames()", capturing_tab);
131f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    ASSERT_TRUE(base::StringToInt(response, &num_frames)) <<
132f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)        "Failed to retrieve frame count: got " << response;
133f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    ASSERT_NE(0, num_frames) << "Failed to capture any frames.";
134f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
135f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    for (int i = 0; i < num_frames; i++) {
136f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      std::string base64_encoded_frame =
137f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)          ExecuteJavascript(base::StringPrintf("getOneCapturedFrame(%d)", i),
138f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)                            capturing_tab);
139f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      std::string decoded_frame;
140f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      ASSERT_TRUE(base::Base64Decode(base64_encoded_frame, &decoded_frame))
141f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)          << "Failed to decode frame data '" << base64_encoded_frame << "'.";
142f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
143f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      std::string file_name = base::StringPrintf("frame_%04d", i);
144f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      base::File frame_file(GetWorkingDir().AppendASCII(file_name),
145f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)                            base::File::FLAG_CREATE | base::File::FLAG_WRITE);
146f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      size_t written = frame_file.Write(0, decoded_frame.c_str(),
147f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)                                        decoded_frame.length());
148f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      ASSERT_EQ(decoded_frame.length(), written);
149424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    }
150eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
151eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
152eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
153eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // which should contain frames of size |width| x |height|.
154eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  //
155eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
156eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // should be build prior to running this test. The resulting binary should
157eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // live next to Chrome.
158424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  bool RunARGBtoI420Converter(int width,
159eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch                              int height,
160eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch                              const base::FilePath& captured_video_filename) {
161eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
162eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
163424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)
164424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    if (!base::PathExists(path_to_converter)) {
165424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
166424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)          << path_to_converter.value();
167424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      return false;
168424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    }
169eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
170eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    CommandLine converter_command(path_to_converter);
171eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
172eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    converter_command.AppendSwitchPath("--output_file",
173eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch                                       captured_video_filename);
174eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    converter_command.AppendSwitchASCII("--width",
175eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch                                        base::StringPrintf("%d", width));
176eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    converter_command.AppendSwitchASCII("--height",
177eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch                                        base::StringPrintf("%d", height));
1780529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    converter_command.AppendSwitchASCII("--delete_frames", "true");
179eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
180eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    // We produce an output file that will later be used as an input to the
181eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    // barcode decoder and frame analyzer tools.
182f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    VLOG(0) << "Running " << converter_command.GetCommandLineString();
183eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    std::string result;
184424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    bool ok = base::GetAppOutput(converter_command, &result);
185f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    VLOG(0) << "Output was:\n\n" << result;
186424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    return ok;
187eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
188eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
189eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // Compares the |captured_video_filename| with the |reference_video_filename|.
190eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  //
191eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // The barcode decoder decodes the captured video containing barcodes overlaid
192eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // into every frame of the video (produced by rgba_to_i420_converter). It
193eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // produces a set of PNG images and a |stats_file| that maps each captured
194eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  // frame to a frame in the reference video. The frames should be of size
195d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)  // |width| x |height|.
196d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)  // All measurements calculated are printed as perf parsable numbers to stdout.
197d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)  bool CompareVideosAndPrintResult(
1985c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu      const char* test_label,
199d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)      int width,
200d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)      int height,
201d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)      const base::FilePath& captured_video_filename,
202d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)      const base::FilePath& reference_video_filename,
203d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)      const base::FilePath& stats_file) {
204424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)
205eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
206eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch        GetBrowserDir().Append(kFrameAnalyzerExecutable));
207eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    base::FilePath path_to_compare_script = GetSourceDir().Append(
208eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
209eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
210424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    if (!base::PathExists(path_to_analyzer)) {
211424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      LOG(ERROR) << "Missing frame analyzer: should be in "
212424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)          << path_to_analyzer.value();
213424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      return false;
214424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    }
215424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    if (!base::PathExists(path_to_compare_script)) {
216424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      LOG(ERROR) << "Missing video compare script: should be in "
217424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)          << path_to_compare_script.value();
218424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      return false;
219424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    }
220eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
2213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    // Note: don't append switches to this command since it will mess up the
2223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    // -u in the python invocation!
2233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    CommandLine compare_command(CommandLine::NO_PROGRAM);
2243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    EXPECT_TRUE(GetPythonCommand(&compare_command));
2253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
2263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArgPath(path_to_compare_script);
2275c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
2283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--ref_video");
2293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArgPath(reference_video_filename);
2303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--test_video");
2313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArgPath(captured_video_filename);
2323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--frame_analyzer");
2333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArgPath(path_to_analyzer);
2343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--yuv_frame_width");
2353551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg(base::StringPrintf("%d", width));
2363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--yuv_frame_height");
2373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg(base::StringPrintf("%d", height));
2383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArg("--stats_file");
2393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    compare_command.AppendArgPath(stats_file);
240eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
241f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    VLOG(0) << "Running " << compare_command.GetCommandLineString();
242d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)    std::string output;
243d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)    bool ok = base::GetAppOutput(compare_command, &output);
244d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)    // Print to stdout to ensure the perf numbers are parsed properly by the
245d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)    // buildbot step.
246d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)    printf("Output was:\n\n%s\n", output.c_str());
247424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    return ok;
248eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
249eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
2505c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu protected:
2515c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu  VideoQualityTestConfig test_config_;
252424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)
253f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  base::FilePath GetWorkingDir() {
254f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    return temp_working_dir_.path();
255f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  }
256f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
257eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch private:
258eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  base::FilePath GetSourceDir() {
259eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    base::FilePath source_dir;
260eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
261eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    return source_dir;
262eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
263eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
264eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  base::FilePath GetBrowserDir() {
265eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    base::FilePath browser_dir;
266eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
267eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    return browser_dir;
268eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  }
269eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
270eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  scoped_ptr<base::Environment> environment_;
271a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  base::FilePath webrtc_reference_video_y4m_;
272f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  base::ScopedTempDir temp_working_dir_;
273eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch};
274eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
2755c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo LiuINSTANTIATE_TEST_CASE_P(
2765c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    WebRtcVideoQualityBrowserTests,
2775c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    WebRtcVideoQualityBrowserTest,
2785c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu    testing::ValuesIn(kVideoConfigurations));
2795c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu
2805c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo LiuIN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
2815c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu                       MANUAL_TestVideoQuality) {
282cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  if (OnWinXp())
283cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    return;  // Fails on XP. http://crbug.com/353078.
284a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
285a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
286a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)      "This is a long-running test; you must specify "
287a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)      "--ui-test-action-max-timeout to have a value of at least 150000.";
288f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
289424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
290eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
291eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  content::WebContents* left_tab =
292a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      OpenPageAndGetUserMediaInNewTabWithConstraints(
293a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
2945c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu          test_config_.constraints);
295eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  content::WebContents* right_tab =
296a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      OpenPageAndGetUserMediaInNewTabWithConstraints(
297cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
2985c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu          test_config_.constraints);
299eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
300010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  SetupPeerconnectionWithLocalStream(left_tab);
301010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  SetupPeerconnectionWithLocalStream(right_tab);
302010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)
303010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles)  NegotiateCall(left_tab, right_tab);
304eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
305a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  // Poll slower here to avoid flooding the log with messages: capturing and
306a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  // sending frames take quite a bit of time.
307a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  int polling_interval_msec = 1000;
308a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
309e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch  EXPECT_TRUE(test::PollingWaitUntil(
310a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      "doneFrameCapturing()", "done-capturing", right_tab,
311a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      polling_interval_msec));
312eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
313eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch  HangUp(left_tab);
314eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
315f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  WriteCapturedFramesToWorkingDir(right_tab);
316eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch
317f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  // Shut everything down to avoid having the javascript race with the analysis
318f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  // tools. For instance, dont have console log printouts interleave with the
319f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  // RESULT lines from the analysis tools (crbug.com/323200).
320f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  chrome::CloseWebContents(browser(), left_tab, false);
321f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  chrome::CloseWebContents(browser(), right_tab, false);
322f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
323f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  ASSERT_TRUE(RunARGBtoI420Converter(
3245c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu      test_config_.width, test_config_.height,
325f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      GetWorkingDir().Append(kCapturedYuvFileName)));
326a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  ASSERT_TRUE(CompareVideosAndPrintResult(
3275c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu      test_config_.test_name,
3285c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu      test_config_.width,
3295c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu      test_config_.height,
330a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      GetWorkingDir().Append(kCapturedYuvFileName),
331c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch      test::GetReferenceFilesDir()
3325c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu          .Append(test_config_.reference_video)
333e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch          .AddExtension(test::kYuvFileExtension),
334a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      GetWorkingDir().Append(kStatsFileName)));
335eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch}
336