chrome_webrtc_video_quality_browsertest.cc revision bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "base/environment.h" 6#include "base/file_util.h" 7#include "base/path_service.h" 8#include "base/process/launch.h" 9#include "base/strings/string_split.h" 10#include "base/strings/stringprintf.h" 11#include "base/test/test_timeouts.h" 12#include "base/time/time.h" 13#include "chrome/browser/chrome_notification_types.h" 14#include "chrome/browser/infobars/infobar.h" 15#include "chrome/browser/infobars/infobar_service.h" 16#include "chrome/browser/media/media_stream_infobar_delegate.h" 17#include "chrome/browser/media/webrtc_browsertest_common.h" 18#include "chrome/browser/profiles/profile.h" 19#include "chrome/browser/ui/browser.h" 20#include "chrome/browser/ui/browser_tabstrip.h" 21#include "chrome/browser/ui/tabs/tab_strip_model.h" 22#include "chrome/common/chrome_switches.h" 23#include "chrome/test/base/in_process_browser_test.h" 24#include "chrome/test/base/ui_test_utils.h" 25#include "chrome/test/perf/perf_test.h" 26#include "chrome/test/ui/ui_test.h" 27#include "content/public/browser/notification_service.h" 28#include "content/public/test/browser_test_utils.h" 29#include "net/test/python_utils.h" 30#include "net/test/spawned_test_server/spawned_test_server.h" 31 32static const base::FilePath::CharType kPeerConnectionServer[] = 33#if defined(OS_WIN) 34 FILE_PATH_LITERAL("peerconnection_server.exe"); 35#else 36 FILE_PATH_LITERAL("peerconnection_server"); 37#endif 38 39static const base::FilePath::CharType kFrameAnalyzerExecutable[] = 40#if defined(OS_WIN) 41 FILE_PATH_LITERAL("frame_analyzer.exe"); 42#else 43 FILE_PATH_LITERAL("frame_analyzer"); 44#endif 45 46static const base::FilePath::CharType kArgbToI420ConverterExecutable[] = 47#if defined(OS_WIN) 48 FILE_PATH_LITERAL("rgba_to_i420_converter.exe"); 49#else 50 FILE_PATH_LITERAL("rgba_to_i420_converter"); 51#endif 52 53static const char kHomeEnvName[] = 54#if defined(OS_WIN) 55 "HOMEPATH"; 56#else 57 "HOME"; 58#endif 59 60// The working dir should be in the user's home folder. 61static const base::FilePath::CharType kWorkingDirName[] = 62 FILE_PATH_LITERAL("webrtc_video_quality"); 63static const base::FilePath::CharType kReferenceYuvFileName[] = 64 FILE_PATH_LITERAL("reference_video.yuv"); 65static const base::FilePath::CharType kCapturedYuvFileName[] = 66 FILE_PATH_LITERAL("captured_video.yuv"); 67static const base::FilePath::CharType kStatsFileName[] = 68 FILE_PATH_LITERAL("stats.txt"); 69static const char kMainWebrtcTestHtmlPage[] = 70 "files/webrtc/webrtc_jsep01_test.html"; 71static const char kCapturingWebrtcHtmlPage[] = 72 "files/webrtc/webrtc_video_quality_test.html"; 73static const int kVgaWidth = 640; 74static const int kVgaHeight = 480; 75 76// If you change the port number, don't forget to modify video_extraction.js 77// too! 78static const char kPyWebSocketPortNumber[] = "12221"; 79 80// Test the video quality of the WebRTC output. 81// 82// Prerequisites: This test case must run on a machine with a virtual webcam 83// that plays video from the reference file located in <the running users home 84// folder>/kWorkingDirName/kReferenceYuvFileName. 85// 86// You must also compile the chromium_builder_webrtc target before you run this 87// test to get all the tools built. 88// 89// The external compare_videos.py script also depends on two external 90// executables which must be located in the PATH when running this test. 91// * zxing (see the CPP version at https://code.google.com/p/zxing) 92// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org) 93// 94// The test case will launch a custom binary (peerconnection_server) which will 95// allow two WebRTC clients to find each other. 96// 97// The test also runs several other custom binaries - rgba_to_i420 converter and 98// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The 99// test also runs a stand alone Python implementation of a WebSocket server 100// (pywebsocket) and a barcode_decoder script. 101class WebrtcVideoQualityBrowserTest : public InProcessBrowserTest { 102 public: 103 WebrtcVideoQualityBrowserTest() 104 : peerconnection_server_(0), 105 pywebsocket_server_(0), 106 environment_(base::Environment::Create()) {} 107 108 virtual void SetUp() OVERRIDE { 109 RunPeerConnectionServer(); 110 InProcessBrowserTest::SetUp(); 111 112 // Ensure we have the stuff we need. 113 EXPECT_TRUE(base::PathExists(GetWorkingDir())) 114 << "Cannot find the working directory for the reference video and " 115 "the temporary files:" << GetWorkingDir().value(); 116 base::FilePath reference_file = 117 GetWorkingDir().Append(kReferenceYuvFileName); 118 EXPECT_TRUE(base::PathExists(reference_file)) 119 << "Cannot find the reference file to be used for video quality " 120 << "comparison: " << reference_file.value(); 121 } 122 123 virtual void TearDown() OVERRIDE { 124 ShutdownPeerConnectionServer(); 125 InProcessBrowserTest::TearDown(); 126 ShutdownPyWebSocketServer(); 127 } 128 129 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 130 // TODO(phoglund): check that user actually has the requisite devices and 131 // print a nice message if not; otherwise the test just times out which can 132 // be confusing. 133 // This test expects real device handling and requires a real webcam / audio 134 // device; it will not work with fake devices. 135 EXPECT_FALSE( 136 command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream)) 137 << "You cannot run this test with fake devices."; 138 } 139 140 void StartPyWebSocketServer() { 141 base::FilePath path_pywebsocket_dir = 142 GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src")); 143 base::FilePath pywebsocket_server = path_pywebsocket_dir.Append( 144 FILE_PATH_LITERAL("mod_pywebsocket/standalone.py")); 145 base::FilePath path_to_data_handler = 146 GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional")); 147 148 EXPECT_TRUE(base::PathExists(pywebsocket_server)) 149 << "Fatal: missing pywebsocket server."; 150 EXPECT_TRUE(base::PathExists(path_to_data_handler)) 151 << "Fatal: missing data handler for pywebsocket server."; 152 153 AppendToPythonPath(path_pywebsocket_dir); 154 CommandLine pywebsocket_command = MakePythonCommand(pywebsocket_server); 155 156 // Construct the command line manually, the server doesn't support -arg=val. 157 pywebsocket_command.AppendArg("-p"); 158 pywebsocket_command.AppendArg(kPyWebSocketPortNumber); 159 pywebsocket_command.AppendArg("-d"); 160 pywebsocket_command.AppendArgPath(path_to_data_handler); 161 162 LOG(INFO) << "Running " << pywebsocket_command.GetCommandLineString(); 163 EXPECT_TRUE(base::LaunchProcess( 164 pywebsocket_command, base::LaunchOptions(), &pywebsocket_server_)) 165 << "Failed to launch pywebsocket server."; 166 } 167 168 void ShutdownPyWebSocketServer() { 169 EXPECT_TRUE(base::KillProcess(pywebsocket_server_, 0, false)) 170 << "Failed to shut down pywebsocket server!"; 171 } 172 173 // Convenience method which executes the provided javascript in the context 174 // of the provided web contents and returns what it evaluated to. 175 std::string ExecuteJavascript(const std::string& javascript, 176 content::WebContents* tab_contents) { 177 std::string result; 178 EXPECT_TRUE(content::ExecuteScriptAndExtractString( 179 tab_contents, javascript, &result)); 180 return result; 181 } 182 183 void GetUserMedia(content::WebContents* tab_contents) { 184 content::WindowedNotificationObserver infobar_added( 185 chrome::NOTIFICATION_TAB_CONTENTS_INFOBAR_ADDED, 186 content::NotificationService::AllSources()); 187 188 // Request user media: this will launch the media stream info bar. 189 EXPECT_EQ("ok-requested", 190 ExecuteJavascript("getUserMedia('{video: true, audio: true}');", 191 tab_contents)); 192 193 // Wait for the bar to pop up, then accept. 194 infobar_added.Wait(); 195 content::Details<InfoBarAddedDetails> details(infobar_added.details()); 196 MediaStreamInfoBarDelegate* media_infobar = 197 details->AsMediaStreamInfoBarDelegate(); 198 media_infobar->Accept(); 199 200 // Wait for WebRTC to call the success callback. 201 EXPECT_TRUE(PollingWaitUntil( 202 "obtainGetUserMediaResult();", "ok-got-stream", tab_contents)); 203 } 204 205 // Ensures we didn't get any errors asynchronously (e.g. while no javascript 206 // call from this test was outstanding). 207 // TODO(phoglund): this becomes obsolete when we switch to communicating with 208 // the DOM message queue. 209 void AssertNoAsynchronousErrors(content::WebContents* tab_contents) { 210 EXPECT_EQ("ok-no-errors", 211 ExecuteJavascript("getAnyTestFailures()", tab_contents)); 212 } 213 214 // The peer connection server lets our two tabs find each other and talk to 215 // each other (e.g. it is the application-specific "signaling solution"). 216 void ConnectToPeerConnectionServer(const std::string peer_name, 217 content::WebContents* tab_contents) { 218 std::string javascript = base::StringPrintf( 219 "connect('http://localhost:8888', '%s');", peer_name.c_str()); 220 EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents)); 221 } 222 223 void EstablishCall(content::WebContents* from_tab, 224 content::WebContents* to_tab) { 225 EXPECT_EQ("ok-peerconnection-created", 226 ExecuteJavascript("preparePeerConnection()", from_tab)); 227 EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab)); 228 EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab)); 229 230 // Ensure the call gets up on both sides. 231 EXPECT_TRUE(PollingWaitUntil( 232 "getPeerConnectionReadyState()", "active", from_tab)); 233 EXPECT_TRUE(PollingWaitUntil( 234 "getPeerConnectionReadyState()", "active", to_tab)); 235 } 236 237 void HangUp(content::WebContents* from_tab) { 238 EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab)); 239 } 240 241 void WaitUntilHangupVerified(content::WebContents* tab_contents) { 242 EXPECT_TRUE(PollingWaitUntil( 243 "getPeerConnectionReadyState()", "no-peer-connection", tab_contents)); 244 } 245 246 // Runs the RGBA to I420 converter on the video in |capture_video_filename|, 247 // which should contain frames of size |width| x |height|. 248 // 249 // The rgba_to_i420_converter is part of the webrtc_test_tools target which 250 // should be build prior to running this test. The resulting binary should 251 // live next to Chrome. 252 void RunARGBtoI420Converter(int width, 253 int height, 254 const base::FilePath& captured_video_filename) { 255 base::FilePath path_to_converter = base::MakeAbsoluteFilePath( 256 GetBrowserDir().Append(kArgbToI420ConverterExecutable)); 257 EXPECT_TRUE(base::PathExists(path_to_converter)) 258 << "Missing ARGB->I420 converter: should be in " 259 << path_to_converter.value(); 260 261 CommandLine converter_command(path_to_converter); 262 converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir()); 263 converter_command.AppendSwitchPath("--output_file", 264 captured_video_filename); 265 converter_command.AppendSwitchASCII("--width", 266 base::StringPrintf("%d", width)); 267 converter_command.AppendSwitchASCII("--height", 268 base::StringPrintf("%d", height)); 269 270 // We produce an output file that will later be used as an input to the 271 // barcode decoder and frame analyzer tools. 272 LOG(INFO) << "Running " << converter_command.GetCommandLineString(); 273 std::string result; 274 EXPECT_TRUE(base::GetAppOutput(converter_command, &result)); 275 LOG(INFO) << "Output was:\n\n" << result; 276 } 277 278 // Compares the |captured_video_filename| with the |reference_video_filename|. 279 // 280 // The barcode decoder decodes the captured video containing barcodes overlaid 281 // into every frame of the video (produced by rgba_to_i420_converter). It 282 // produces a set of PNG images and a |stats_file| that maps each captured 283 // frame to a frame in the reference video. The frames should be of size 284 // |width| x |height|. The output of compare_videos.py is returned. 285 std::string CompareVideos(int width, 286 int height, 287 const base::FilePath& captured_video_filename, 288 const base::FilePath& reference_video_filename, 289 const base::FilePath& stats_file) { 290 base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath( 291 GetBrowserDir().Append(kFrameAnalyzerExecutable)); 292 base::FilePath path_to_compare_script = GetSourceDir().Append( 293 FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py")); 294 295 EXPECT_TRUE(base::PathExists(path_to_analyzer)) 296 << "Missing frame analyzer: should be in " << path_to_analyzer.value(); 297 EXPECT_TRUE(base::PathExists(path_to_compare_script)) 298 << "Missing video compare script: should be in " 299 << path_to_compare_script.value(); 300 301 CommandLine compare_command = MakePythonCommand(path_to_compare_script); 302 compare_command.AppendSwitchPath("--ref_video", reference_video_filename); 303 compare_command.AppendSwitchPath("--test_video", captured_video_filename); 304 compare_command.AppendSwitchPath("--frame_analyzer", path_to_analyzer); 305 compare_command.AppendSwitchASCII("--yuv_frame_width", 306 base::StringPrintf("%d", width)); 307 compare_command.AppendSwitchASCII("--yuv_frame_height", 308 base::StringPrintf("%d", height)); 309 compare_command.AppendSwitchPath("--stats_file", stats_file); 310 311 LOG(INFO) << "Running " << compare_command.GetCommandLineString(); 312 std::string result; 313 EXPECT_TRUE(base::GetAppOutput(compare_command, &result)); 314 LOG(INFO) << "Output was:\n\n" << result; 315 return result; 316 } 317 318 // Processes the |frame_analyzer_output| for the different frame counts. 319 // 320 // The frame analyzer outputs additional information about the number of 321 // unique frames captured, The max number of repeated frames in a sequence and 322 // the max number of skipped frames. These values are then written to the Perf 323 // Graph. (Note: Some of the repeated or skipped frames will probably be due 324 // to the imperfection of JavaScript timers). 325 void PrintFramesCountPerfResults(std::string frame_analyzer_output) { 326 size_t unique_frames_pos = 327 frame_analyzer_output.rfind("Unique_frames_count"); 328 EXPECT_NE(unique_frames_pos, std::string::npos) 329 << "Missing Unique_frames_count in frame analyzer output:\n" 330 << frame_analyzer_output; 331 332 std::string unique_frame_counts = 333 frame_analyzer_output.substr(unique_frames_pos); 334 // TODO(phoglund): Fix ESTATS result to not have this silly newline. 335 std::replace( 336 unique_frame_counts.begin(), unique_frame_counts.end(), '\n', ' '); 337 338 std::vector<std::pair<std::string, std::string> > key_values; 339 base::SplitStringIntoKeyValuePairs( 340 unique_frame_counts, ':', ' ', &key_values); 341 std::vector<std::pair<std::string, std::string> >::const_iterator iter; 342 for (iter = key_values.begin(); iter != key_values.end(); ++iter) { 343 const std::pair<std::string, std::string>& key_value = *iter; 344 perf_test::PrintResult( 345 key_value.first, "", "VGA", key_value.second, "", false); 346 } 347 } 348 349 // Processes the |frame_analyzer_output| to extract the PSNR and SSIM values. 350 // 351 // The frame analyzer produces PSNR and SSIM results for every unique frame 352 // that has been captured. This method forms a list of all the psnr and ssim 353 // values and passes it to PrintResultList() for printing on the Perf Graph. 354 void PrintPsnrAndSsimPerfResults(std::string frame_analyzer_output) { 355 size_t stats_start = frame_analyzer_output.find("BSTATS"); 356 EXPECT_NE(stats_start, std::string::npos) 357 << "Missing BSTATS in frame analyzer output:\n" 358 << frame_analyzer_output; 359 size_t stats_end = frame_analyzer_output.find("ESTATS"); 360 EXPECT_NE(stats_end, std::string::npos) 361 << "Missing ESTATS in frame analyzer output:\n" 362 << frame_analyzer_output; 363 364 stats_start += std::string("BSTATS").size(); 365 std::string psnr_ssim_stats = 366 frame_analyzer_output.substr(stats_start, stats_end - stats_start); 367 368 // PSNR and SSIM values aren't really key-value pairs but it is convenient 369 // to parse them as such. 370 // TODO(phoglund): make the format more convenient so we need less 371 // processing here. 372 std::vector<std::pair<std::string, std::string> > psnr_ssim_entries; 373 base::SplitStringIntoKeyValuePairs( 374 psnr_ssim_stats, ' ', ';', &psnr_ssim_entries); 375 376 std::string psnr_value_list; 377 std::string ssim_value_list; 378 std::vector<std::pair<std::string, std::string> >::const_iterator iter; 379 for (iter = psnr_ssim_entries.begin(); iter != psnr_ssim_entries.end(); 380 ++iter) { 381 const std::pair<std::string, std::string>& psnr_and_ssim = *iter; 382 psnr_value_list.append(psnr_and_ssim.first).append(","); 383 ssim_value_list.append(psnr_and_ssim.second).append(","); 384 } 385 // Nuke last comma. 386 psnr_value_list.erase(psnr_value_list.size() - 1); 387 ssim_value_list.erase(ssim_value_list.size() - 1); 388 389 perf_test::PrintResultList("PSNR", "", "VGA", psnr_value_list, "dB", false); 390 perf_test::PrintResultList("SSIM", "", "VGA", ssim_value_list, "", false); 391 } 392 393 base::FilePath GetWorkingDir() { 394 std::string home_dir; 395 environment_->GetVar(kHomeEnvName, &home_dir); 396 base::FilePath::StringType native_home_dir(home_dir.begin(), 397 home_dir.end()); 398 return base::FilePath(native_home_dir).Append(kWorkingDirName); 399 } 400 401 private: 402 void RunPeerConnectionServer() { 403 // TODO(phoglund): de-dupe later: next line differs from original. 404 base::FilePath peerconnection_server = 405 GetBrowserDir().Append(kPeerConnectionServer); 406 407 EXPECT_TRUE(base::PathExists(peerconnection_server)) 408 << "Missing peerconnection_server. You must build " 409 "it so it ends up next to the browser test binary."; 410 EXPECT_TRUE(base::LaunchProcess(CommandLine(peerconnection_server), 411 base::LaunchOptions(), 412 &peerconnection_server_)) 413 << "Failed to launch peerconnection_server."; 414 } 415 416 void ShutdownPeerConnectionServer() { 417 EXPECT_TRUE(base::KillProcess(peerconnection_server_, 0, false)) 418 << "Failed to shut down peerconnection_server!"; 419 } 420 421 base::FilePath GetSourceDir() { 422 base::FilePath source_dir; 423 PathService::Get(base::DIR_SOURCE_ROOT, &source_dir); 424 return source_dir; 425 } 426 427 base::FilePath GetBrowserDir() { 428 base::FilePath browser_dir; 429 EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir)); 430 return browser_dir; 431 } 432 433 CommandLine MakePythonCommand(base::FilePath python_script) { 434 CommandLine python_command(CommandLine::NO_PROGRAM); 435 EXPECT_TRUE(GetPythonCommand(&python_command)); 436 CommandLine complete_command(python_script); 437 complete_command.PrependWrapper(python_command.GetCommandLineString()); 438 return complete_command; 439 } 440 441 base::ProcessHandle peerconnection_server_; 442 base::ProcessHandle pywebsocket_server_; 443 scoped_ptr<base::Environment> environment_; 444}; 445 446// Broken on Win: failing to start pywebsocket_server. http://crbug.com/255499. 447#define MAYBE_MANUAL_TestVGAVideoQuality DISABLED_MANUAL_TestVGAVideoQuality 448 449IN_PROC_BROWSER_TEST_F(WebrtcVideoQualityBrowserTest, 450 MAYBE_MANUAL_TestVGAVideoQuality) { 451 // TODO(phoglund): de-dupe from chrome_webrtc_browsertest.cc. 452 StartPyWebSocketServer(); 453 454 EXPECT_TRUE(test_server()->Start()); 455 456 ui_test_utils::NavigateToURL(browser(), 457 test_server()->GetURL(kMainWebrtcTestHtmlPage)); 458 content::WebContents* left_tab = 459 browser()->tab_strip_model()->GetActiveWebContents(); 460 GetUserMedia(left_tab); 461 462 chrome::AddBlankTabAt(browser(), -1, true); 463 content::WebContents* right_tab = 464 browser()->tab_strip_model()->GetActiveWebContents(); 465 // TODO(phoglund): (de-dupe later) different from original flow. 466 ui_test_utils::NavigateToURL(browser(), 467 test_server()->GetURL(kCapturingWebrtcHtmlPage)); 468 GetUserMedia(right_tab); 469 470 ConnectToPeerConnectionServer("peer 1", left_tab); 471 ConnectToPeerConnectionServer("peer 2", right_tab); 472 473 EstablishCall(left_tab, right_tab); 474 475 AssertNoAsynchronousErrors(left_tab); 476 AssertNoAsynchronousErrors(right_tab); 477 478 // Poll slower here to avoid flooding the log with messages: capturing and 479 // sending frames take quite a bit of time. 480 int polling_interval_msec = 1000; 481 482 // TODO(phoglund): (de-dupe later) different from original flow. 483 EXPECT_TRUE(PollingWaitUntil( 484 "doneFrameCapturing()", "done-capturing", right_tab, 485 polling_interval_msec)); 486 487 HangUp(left_tab); 488 WaitUntilHangupVerified(left_tab); 489 WaitUntilHangupVerified(right_tab); 490 491 AssertNoAsynchronousErrors(left_tab); 492 AssertNoAsynchronousErrors(right_tab); 493 494 // TODO(phoglund): (de-dupe later) different from original flow. 495 EXPECT_TRUE(PollingWaitUntil( 496 "haveMoreFramesToSend()", "no-more-frames", right_tab, 497 polling_interval_msec)); 498 499 RunARGBtoI420Converter( 500 kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName)); 501 std::string output = 502 CompareVideos(kVgaWidth, 503 kVgaHeight, 504 GetWorkingDir().Append(kCapturedYuvFileName), 505 GetWorkingDir().Append(kReferenceYuvFileName), 506 GetWorkingDir().Append(kStatsFileName)); 507 508 PrintFramesCountPerfResults(output); 509 PrintPsnrAndSsimPerfResults(output); 510} 511