chrome_webrtc_video_quality_browsertest.cc revision 1320f92c476a1ad9d19dba2a48c72b75566198e9
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "base/base64.h" 6#include "base/command_line.h" 7#include "base/environment.h" 8#include "base/files/file.h" 9#include "base/files/file_util.h" 10#include "base/files/scoped_temp_dir.h" 11#include "base/path_service.h" 12#include "base/process/launch.h" 13#include "base/strings/string_number_conversions.h" 14#include "base/strings/string_split.h" 15#include "base/strings/stringprintf.h" 16#include "base/test/test_timeouts.h" 17#include "base/time/time.h" 18#include "chrome/browser/chrome_notification_types.h" 19#include "chrome/browser/infobars/infobar_service.h" 20#include "chrome/browser/media/media_stream_infobar_delegate.h" 21#include "chrome/browser/media/webrtc_browsertest_base.h" 22#include "chrome/browser/media/webrtc_browsertest_common.h" 23#include "chrome/browser/profiles/profile.h" 24#include "chrome/browser/ui/browser.h" 25#include "chrome/browser/ui/browser_tabstrip.h" 26#include "chrome/browser/ui/tabs/tab_strip_model.h" 27#include "chrome/common/chrome_switches.h" 28#include "chrome/test/base/in_process_browser_test.h" 29#include "chrome/test/base/ui_test_utils.h" 30#include "components/infobars/core/infobar.h" 31#include "content/public/browser/notification_service.h" 32#include "content/public/test/browser_test_utils.h" 33#include "media/base/media_switches.h" 34#include "net/test/embedded_test_server/embedded_test_server.h" 35#include "net/test/python_utils.h" 36#include "testing/perf/perf_test.h" 37#include "ui/gl/gl_switches.h" 38 39static const base::FilePath::CharType kFrameAnalyzerExecutable[] = 40#if defined(OS_WIN) 41 FILE_PATH_LITERAL("frame_analyzer.exe"); 42#else 43 FILE_PATH_LITERAL("frame_analyzer"); 44#endif 45 46static const base::FilePath::CharType kArgbToI420ConverterExecutable[] = 47#if defined(OS_WIN) 48 FILE_PATH_LITERAL("rgba_to_i420_converter.exe"); 49#else 50 FILE_PATH_LITERAL("rgba_to_i420_converter"); 51#endif 52 53static const base::FilePath::CharType kCapturedYuvFileName[] = 54 FILE_PATH_LITERAL("captured_video.yuv"); 55static const base::FilePath::CharType kStatsFileName[] = 56 FILE_PATH_LITERAL("stats.txt"); 57static const char kMainWebrtcTestHtmlPage[] = 58 "/webrtc/webrtc_jsep01_test.html"; 59static const char kCapturingWebrtcHtmlPage[] = 60 "/webrtc/webrtc_video_quality_test.html"; 61 62static const struct VideoQualityTestConfig { 63 const char* test_name; 64 int width; 65 int height; 66 const base::FilePath::CharType* reference_video; 67 const char* constraints; 68} kVideoConfigurations[] = { 69 { "360p", 640, 360, 70 test::kReferenceFileName360p, 71 WebRtcTestBase::kAudioVideoCallConstraints360p }, 72 { "720p", 1280, 720, 73 test::kReferenceFileName720p, 74 WebRtcTestBase::kAudioVideoCallConstraints720p }, 75}; 76 77// Test the video quality of the WebRTC output. 78// 79// Prerequisites: This test case must run on a machine with a chrome playing 80// the video from the reference files located in GetReferenceFilesDir(). 81// The file kReferenceY4mFileName.kY4mFileExtension is played using a 82// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for 83// comparison. 84// 85// You must also compile the chromium_builder_webrtc target before you run this 86// test to get all the tools built. 87// 88// The external compare_videos.py script also depends on two external 89// executables which must be located in the PATH when running this test. 90// * zxing (see the CPP version at https://code.google.com/p/zxing) 91// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org) 92// 93// The test runs several custom binaries - rgba_to_i420 converter and 94// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The 95// test also runs a stand alone Python implementation of a WebSocket server 96// (pywebsocket) and a barcode_decoder script. 97class WebRtcVideoQualityBrowserTest : public WebRtcTestBase, 98 public testing::WithParamInterface<VideoQualityTestConfig> { 99 public: 100 WebRtcVideoQualityBrowserTest() 101 : environment_(base::Environment::Create()) { 102 test_config_ = GetParam(); 103 } 104 105 virtual void SetUpInProcessBrowserTestFixture() OVERRIDE { 106 DetectErrorsInJavaScript(); // Look for errors in our rather complex js. 107 108 ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir()); 109 } 110 111 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 112 // Set up the command line option with the expected file name. We will check 113 // its existence in HasAllRequiredResources(). 114 webrtc_reference_video_y4m_ = test::GetReferenceFilesDir() 115 .Append(test_config_.reference_video) 116 .AddExtension(test::kY4mFileExtension); 117 command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture, 118 webrtc_reference_video_y4m_); 119 command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream); 120 121 // The video playback will not work without a GPU, so force its use here. 122 command_line->AppendSwitch(switches::kUseGpuInTests); 123 } 124 125 // Writes all frames we've captured so far by grabbing them from the 126 // javascript and writing them to the temporary work directory. 127 void WriteCapturedFramesToWorkingDir(content::WebContents* capturing_tab) { 128 int num_frames = 0; 129 std::string response = 130 ExecuteJavascript("getTotalNumberCapturedFrames()", capturing_tab); 131 ASSERT_TRUE(base::StringToInt(response, &num_frames)) << 132 "Failed to retrieve frame count: got " << response; 133 ASSERT_NE(0, num_frames) << "Failed to capture any frames."; 134 135 for (int i = 0; i < num_frames; i++) { 136 std::string base64_encoded_frame = 137 ExecuteJavascript(base::StringPrintf("getOneCapturedFrame(%d)", i), 138 capturing_tab); 139 std::string decoded_frame; 140 ASSERT_TRUE(base::Base64Decode(base64_encoded_frame, &decoded_frame)) 141 << "Failed to decode frame data '" << base64_encoded_frame << "'."; 142 143 std::string file_name = base::StringPrintf("frame_%04d", i); 144 base::File frame_file(GetWorkingDir().AppendASCII(file_name), 145 base::File::FLAG_CREATE | base::File::FLAG_WRITE); 146 size_t written = frame_file.Write(0, decoded_frame.c_str(), 147 decoded_frame.length()); 148 ASSERT_EQ(decoded_frame.length(), written); 149 } 150 } 151 152 // Runs the RGBA to I420 converter on the video in |capture_video_filename|, 153 // which should contain frames of size |width| x |height|. 154 // 155 // The rgba_to_i420_converter is part of the webrtc_test_tools target which 156 // should be build prior to running this test. The resulting binary should 157 // live next to Chrome. 158 bool RunARGBtoI420Converter(int width, 159 int height, 160 const base::FilePath& captured_video_filename) { 161 base::FilePath path_to_converter = base::MakeAbsoluteFilePath( 162 GetBrowserDir().Append(kArgbToI420ConverterExecutable)); 163 164 if (!base::PathExists(path_to_converter)) { 165 LOG(ERROR) << "Missing ARGB->I420 converter: should be in " 166 << path_to_converter.value(); 167 return false; 168 } 169 170 CommandLine converter_command(path_to_converter); 171 converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir()); 172 converter_command.AppendSwitchPath("--output_file", 173 captured_video_filename); 174 converter_command.AppendSwitchASCII("--width", 175 base::StringPrintf("%d", width)); 176 converter_command.AppendSwitchASCII("--height", 177 base::StringPrintf("%d", height)); 178 converter_command.AppendSwitchASCII("--delete_frames", "true"); 179 180 // We produce an output file that will later be used as an input to the 181 // barcode decoder and frame analyzer tools. 182 VLOG(0) << "Running " << converter_command.GetCommandLineString(); 183 std::string result; 184 bool ok = base::GetAppOutput(converter_command, &result); 185 VLOG(0) << "Output was:\n\n" << result; 186 return ok; 187 } 188 189 // Compares the |captured_video_filename| with the |reference_video_filename|. 190 // 191 // The barcode decoder decodes the captured video containing barcodes overlaid 192 // into every frame of the video (produced by rgba_to_i420_converter). It 193 // produces a set of PNG images and a |stats_file| that maps each captured 194 // frame to a frame in the reference video. The frames should be of size 195 // |width| x |height|. 196 // All measurements calculated are printed as perf parsable numbers to stdout. 197 bool CompareVideosAndPrintResult( 198 const char* test_label, 199 int width, 200 int height, 201 const base::FilePath& captured_video_filename, 202 const base::FilePath& reference_video_filename, 203 const base::FilePath& stats_file) { 204 205 base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath( 206 GetBrowserDir().Append(kFrameAnalyzerExecutable)); 207 base::FilePath path_to_compare_script = GetSourceDir().Append( 208 FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py")); 209 210 if (!base::PathExists(path_to_analyzer)) { 211 LOG(ERROR) << "Missing frame analyzer: should be in " 212 << path_to_analyzer.value(); 213 return false; 214 } 215 if (!base::PathExists(path_to_compare_script)) { 216 LOG(ERROR) << "Missing video compare script: should be in " 217 << path_to_compare_script.value(); 218 return false; 219 } 220 221 // Note: don't append switches to this command since it will mess up the 222 // -u in the python invocation! 223 CommandLine compare_command(CommandLine::NO_PROGRAM); 224 EXPECT_TRUE(GetPythonCommand(&compare_command)); 225 226 compare_command.AppendArgPath(path_to_compare_script); 227 compare_command.AppendArg(base::StringPrintf("--label=%s", test_label)); 228 compare_command.AppendArg("--ref_video"); 229 compare_command.AppendArgPath(reference_video_filename); 230 compare_command.AppendArg("--test_video"); 231 compare_command.AppendArgPath(captured_video_filename); 232 compare_command.AppendArg("--frame_analyzer"); 233 compare_command.AppendArgPath(path_to_analyzer); 234 compare_command.AppendArg("--yuv_frame_width"); 235 compare_command.AppendArg(base::StringPrintf("%d", width)); 236 compare_command.AppendArg("--yuv_frame_height"); 237 compare_command.AppendArg(base::StringPrintf("%d", height)); 238 compare_command.AppendArg("--stats_file"); 239 compare_command.AppendArgPath(stats_file); 240 241 VLOG(0) << "Running " << compare_command.GetCommandLineString(); 242 std::string output; 243 bool ok = base::GetAppOutput(compare_command, &output); 244 // Print to stdout to ensure the perf numbers are parsed properly by the 245 // buildbot step. 246 printf("Output was:\n\n%s\n", output.c_str()); 247 return ok; 248 } 249 250 protected: 251 VideoQualityTestConfig test_config_; 252 253 base::FilePath GetWorkingDir() { 254 return temp_working_dir_.path(); 255 } 256 257 private: 258 base::FilePath GetSourceDir() { 259 base::FilePath source_dir; 260 PathService::Get(base::DIR_SOURCE_ROOT, &source_dir); 261 return source_dir; 262 } 263 264 base::FilePath GetBrowserDir() { 265 base::FilePath browser_dir; 266 EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir)); 267 return browser_dir; 268 } 269 270 scoped_ptr<base::Environment> environment_; 271 base::FilePath webrtc_reference_video_y4m_; 272 base::ScopedTempDir temp_working_dir_; 273}; 274 275INSTANTIATE_TEST_CASE_P( 276 WebRtcVideoQualityBrowserTests, 277 WebRtcVideoQualityBrowserTest, 278 testing::ValuesIn(kVideoConfigurations)); 279 280IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest, 281 MANUAL_TestVideoQuality) { 282 if (OnWinXp()) 283 return; // Fails on XP. http://crbug.com/353078. 284 285 ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) << 286 "This is a long-running test; you must specify " 287 "--ui-test-action-max-timeout to have a value of at least 150000."; 288 ASSERT_TRUE(test::HasReferenceFilesInCheckout()); 289 ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady()); 290 291 content::WebContents* left_tab = 292 OpenPageAndGetUserMediaInNewTabWithConstraints( 293 embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage), 294 test_config_.constraints); 295 content::WebContents* right_tab = 296 OpenPageAndGetUserMediaInNewTabWithConstraints( 297 embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage), 298 test_config_.constraints); 299 300 SetupPeerconnectionWithLocalStream(left_tab); 301 SetupPeerconnectionWithLocalStream(right_tab); 302 303 NegotiateCall(left_tab, right_tab); 304 305 // Poll slower here to avoid flooding the log with messages: capturing and 306 // sending frames take quite a bit of time. 307 int polling_interval_msec = 1000; 308 309 EXPECT_TRUE(test::PollingWaitUntil( 310 "doneFrameCapturing()", "done-capturing", right_tab, 311 polling_interval_msec)); 312 313 HangUp(left_tab); 314 315 WriteCapturedFramesToWorkingDir(right_tab); 316 317 // Shut everything down to avoid having the javascript race with the analysis 318 // tools. For instance, dont have console log printouts interleave with the 319 // RESULT lines from the analysis tools (crbug.com/323200). 320 chrome::CloseWebContents(browser(), left_tab, false); 321 chrome::CloseWebContents(browser(), right_tab, false); 322 323 ASSERT_TRUE(RunARGBtoI420Converter( 324 test_config_.width, test_config_.height, 325 GetWorkingDir().Append(kCapturedYuvFileName))); 326 ASSERT_TRUE(CompareVideosAndPrintResult( 327 test_config_.test_name, 328 test_config_.width, 329 test_config_.height, 330 GetWorkingDir().Append(kCapturedYuvFileName), 331 test::GetReferenceFilesDir() 332 .Append(test_config_.reference_video) 333 .AddExtension(test::kYuvFileExtension), 334 GetWorkingDir().Append(kStatsFileName))); 335} 336