chrome_webrtc_audio_quality_browsertest.cc revision 424c4d7b64af9d0d8fd9624f381f469654d5e3d2
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <ctime> 6 7#include "base/command_line.h" 8#include "base/file_util.h" 9#include "base/path_service.h" 10#include "base/process/launch.h" 11#include "base/scoped_native_library.h" 12#include "base/strings/stringprintf.h" 13#include "base/win/windows_version.h" 14#include "chrome/browser/media/webrtc_browsertest_base.h" 15#include "chrome/browser/media/webrtc_browsertest_common.h" 16#include "chrome/browser/profiles/profile.h" 17#include "chrome/browser/ui/browser.h" 18#include "chrome/browser/ui/browser_tabstrip.h" 19#include "chrome/browser/ui/tabs/tab_strip_model.h" 20#include "chrome/common/chrome_paths.h" 21#include "chrome/common/chrome_switches.h" 22#include "chrome/test/base/ui_test_utils.h" 23#include "chrome/test/perf/perf_test.h" 24#include "chrome/test/ui/ui_test.h" 25#include "content/public/test/browser_test_utils.h" 26#include "net/test/embedded_test_server/embedded_test_server.h" 27 28static const base::FilePath::CharType kPeerConnectionServer[] = 29#if defined(OS_WIN) 30 FILE_PATH_LITERAL("peerconnection_server.exe"); 31#else 32 FILE_PATH_LITERAL("peerconnection_server"); 33#endif 34 35static const base::FilePath::CharType kReferenceFile[] = 36#if defined (OS_WIN) 37 FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-win.wav"); 38#else 39 FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-linux.wav"); 40#endif 41 42// The javascript will load the reference file relative to its location, 43// which is in /webrtc on the web server. Therefore, prepend a '..' traversal. 44static const char kReferenceFileRelativeUrl[] = 45#if defined (OS_WIN) 46 "../pyauto_private/webrtc/human-voice-win.wav"; 47#else 48 "../pyauto_private/webrtc/human-voice-linux.wav"; 49#endif 50 51static const base::FilePath::CharType kToolsPath[] = 52 FILE_PATH_LITERAL("pyauto_private/media/tools"); 53 54static const char kMainWebrtcTestHtmlPage[] = 55 "files/webrtc/webrtc_audio_quality_test.html"; 56 57base::FilePath GetTestDataDir() { 58 base::FilePath source_dir; 59 PathService::Get(chrome::DIR_TEST_DATA, &source_dir); 60 return source_dir; 61} 62 63// Test we can set up a WebRTC call and play audio through it. 64// 65// This test will only work on machines that have been configured to record 66// their own input. 67// 68// On Linux: 69// 1. # sudo apt-get install pavucontrol 70// 2. For the user who will run the test: # pavucontrol 71// 3. In a separate terminal, # arecord dummy 72// 4. In pavucontrol, go to the recording tab. 73// 5. For the ALSA plug-in [aplay]: ALSA Capture from, change from <x> to 74// <Monitor of x>, where x is whatever your primary sound device is called. 75// This test expects the device id to be render.monitor - if it's something 76// else, the microphone level will not get forced to 100% appropriately. 77// See ForceMicrophoneVolumeTo100% for more details. You can list the 78// available monitor devices on your system by running the command 79// pacmd list-sources | grep name | grep monitor. 80// 6. Try launching chrome as the target user on the target machine, try 81// playing, say, a YouTube video, and record with # arecord -f dat tmp.dat. 82// Verify the recording with aplay (should have recorded what you played 83// from chrome). 84// 85// On Windows 7: 86// 1. Control panel > Sound > Manage audio devices. 87// 2. In the recording tab, right-click in an empty space in the pane with the 88// devices. Tick 'show disabled devices'. 89// 3. You should see a 'stero mix' device - this is what your speakers output. 90// Right click > Properties. 91// 4. In the Listen tab for the mix device, check the 'listen to this device' 92// checkbox. Ensure the mix device is the default recording device. 93// 5. Launch chrome and try playing a video with sound. You should see 94// in the volume meter for the mix device. Configure the mix device to have 95// 50 / 100 in level. Also go into the playback tab, right-click Speakers, 96// and set that level to 50 / 100. Otherwise you will get distortion in 97// the recording. 98class WebrtcAudioQualityBrowserTest : public WebRtcTestBase { 99 public: 100 virtual void SetUpInProcessBrowserTestFixture() OVERRIDE { 101 PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem(); 102 } 103 104 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 105 // TODO(phoglund): check that user actually has the requisite devices and 106 // print a nice message if not; otherwise the test just times out which can 107 // be confusing. 108 // This test expects real device handling and requires a real webcam / audio 109 // device; it will not work with fake devices. 110 EXPECT_FALSE(command_line->HasSwitch( 111 switches::kUseFakeDeviceForMediaStream)); 112 EXPECT_FALSE(command_line->HasSwitch( 113 switches::kUseFakeUIForMediaStream)); 114 } 115 116 bool HasAllRequiredResources() { 117 base::FilePath reference_file = 118 GetTestDataDir().Append(kReferenceFile); 119 if (!base::PathExists(reference_file)) { 120 LOG(ERROR) << "Cannot find the reference file to be used for audio " 121 << "quality comparison: " << reference_file.value(); 122 return false; 123 } 124 return true; 125 } 126 127 void AddAudioFile(const std::string& input_file_relative_url, 128 content::WebContents* tab_contents) { 129 EXPECT_EQ("ok-added", ExecuteJavascript( 130 "addAudioFile('" + input_file_relative_url + "')", tab_contents)); 131 } 132 133 void PlayAudioFile(content::WebContents* tab_contents) { 134 EXPECT_EQ("ok-playing", ExecuteJavascript("playAudioFile()", tab_contents)); 135 } 136 137 // Convenience method which executes the provided javascript in the context 138 // of the provided web contents and returns what it evaluated to. 139 std::string ExecuteJavascript(const std::string& javascript, 140 content::WebContents* tab_contents) { 141 std::string result; 142 EXPECT_TRUE(content::ExecuteScriptAndExtractString( 143 tab_contents, javascript, &result)); 144 return result; 145 } 146 147 // Ensures we didn't get any errors asynchronously (e.g. while no javascript 148 // call from this test was outstanding). 149 // TODO(phoglund): this becomes obsolete when we switch to communicating with 150 // the DOM message queue. 151 void AssertNoAsynchronousErrors(content::WebContents* tab_contents) { 152 EXPECT_EQ("ok-no-errors", 153 ExecuteJavascript("getAnyTestFailures()", tab_contents)); 154 } 155 156 // The peer connection server lets our two tabs find each other and talk to 157 // each other (e.g. it is the application-specific "signaling solution"). 158 void ConnectToPeerConnectionServer(const std::string peer_name, 159 content::WebContents* tab_contents) { 160 std::string javascript = base::StringPrintf( 161 "connect('http://localhost:8888', '%s');", peer_name.c_str()); 162 EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents)); 163 } 164 165 void EstablishCall(content::WebContents* from_tab, 166 content::WebContents* to_tab) { 167 EXPECT_EQ("ok-negotiating", 168 ExecuteJavascript("negotiateCall()", from_tab)); 169 170 // Ensure the call gets up on both sides. 171 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 172 "active", from_tab)); 173 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 174 "active", to_tab)); 175 } 176 177 void HangUp(content::WebContents* from_tab) { 178 EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab)); 179 } 180 181 void WaitUntilHangupVerified(content::WebContents* tab_contents) { 182 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 183 "no-peer-connection", tab_contents)); 184 } 185 186 base::FilePath CreateTemporaryWaveFile() { 187 base::FilePath filename; 188 EXPECT_TRUE(file_util::CreateTemporaryFile(&filename)); 189 base::FilePath wav_filename = 190 filename.AddExtension(FILE_PATH_LITERAL(".wav")); 191 EXPECT_TRUE(base::Move(filename, wav_filename)); 192 return wav_filename; 193 } 194 195 PeerConnectionServerRunner peerconnection_server_; 196}; 197 198class AudioRecorder { 199 public: 200 AudioRecorder(): recording_application_(base::kNullProcessHandle) {} 201 ~AudioRecorder() {} 202 203 // Starts the recording program for the specified duration. Returns true 204 // on success. 205 bool StartRecording(int duration_sec, const base::FilePath& output_file, 206 bool mono) { 207 EXPECT_EQ(base::kNullProcessHandle, recording_application_) 208 << "Tried to record, but is already recording."; 209 210 CommandLine command_line(CommandLine::NO_PROGRAM); 211#if defined(OS_WIN) 212 // This disable is required to run SoundRecorder.exe on 64-bit Windows 213 // from a 32-bit binary. We need to load the wow64 disable function from 214 // the DLL since it doesn't exist on Windows XP. 215 // TODO(phoglund): find some cleaner solution than using SoundRecorder.exe. 216 base::ScopedNativeLibrary kernel32_lib(base::FilePath(L"kernel32")); 217 if (kernel32_lib.is_valid()) { 218 typedef BOOL (WINAPI* Wow64DisableWow64FSRedirection)(PVOID*); 219 Wow64DisableWow64FSRedirection wow_64_disable_wow_64_fs_redirection; 220 wow_64_disable_wow_64_fs_redirection = 221 reinterpret_cast<Wow64DisableWow64FSRedirection>( 222 kernel32_lib.GetFunctionPointer( 223 "Wow64DisableWow64FsRedirection")); 224 if (wow_64_disable_wow_64_fs_redirection != NULL) { 225 PVOID* ignored = NULL; 226 wow_64_disable_wow_64_fs_redirection(ignored); 227 } 228 } 229 230 char duration_in_hms[128] = {0}; 231 struct tm duration_tm = {0}; 232 duration_tm.tm_sec = duration_sec; 233 EXPECT_NE(0u, strftime(duration_in_hms, arraysize(duration_in_hms), 234 "%H:%M:%S", &duration_tm)); 235 236 command_line.SetProgram( 237 base::FilePath(FILE_PATH_LITERAL("SoundRecorder.exe"))); 238 command_line.AppendArg("/FILE"); 239 command_line.AppendArgPath(output_file); 240 command_line.AppendArg("/DURATION"); 241 command_line.AppendArg(duration_in_hms); 242#else 243 int num_channels = mono ? 1 : 2; 244 command_line.SetProgram(base::FilePath("arecord")); 245 command_line.AppendArg("-d"); 246 command_line.AppendArg(base::StringPrintf("%d", duration_sec)); 247 command_line.AppendArg("-f"); 248 command_line.AppendArg("dat"); 249 command_line.AppendArg("-c"); 250 command_line.AppendArg(base::StringPrintf("%d", num_channels)); 251 command_line.AppendArgPath(output_file); 252#endif 253 254 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 255 return base::LaunchProcess(command_line, base::LaunchOptions(), 256 &recording_application_); 257 } 258 259 // Joins the recording program. Returns true on success. 260 bool WaitForRecordingToEnd() { 261 int exit_code = -1; 262 base::WaitForExitCode(recording_application_, &exit_code); 263 return exit_code == 0; 264 } 265 private: 266 base::ProcessHandle recording_application_; 267}; 268 269void ForceMicrophoneVolumeTo100Percent() { 270#if defined(OS_WIN) 271 CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append( 272 FILE_PATH_LITERAL("force_mic_volume_max.exe"))); 273#else 274 const std::string kRecordingDeviceId = "render.monitor"; 275 const std::string kHundredPercentVolume = "65536"; 276 277 CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("pacmd"))); 278 command_line.AppendArg("set-source-volume"); 279 command_line.AppendArg(kRecordingDeviceId); 280 command_line.AppendArg(kHundredPercentVolume); 281#endif 282 283 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 284 std::string result; 285 if (!base::GetAppOutput(command_line, &result)) { 286 // It's hard to figure out for instance the default PA recording device name 287 // for different systems, so just warn here. Users will most often have a 288 // reasonable mic level on their systems. 289 LOG(WARNING) << "Failed to set mic volume to 100% on your system. " << 290 "The test may fail or have results distorted; please ensure that " << 291 "your mic level is 100% manually."; 292 } 293} 294 295// Removes silence from beginning and end of the |input_audio_file| and writes 296// the result to the |output_audio_file|. Returns true on success. 297bool RemoveSilence(const base::FilePath& input_file, 298 const base::FilePath& output_file) { 299 // SOX documentation for silence command: http://sox.sourceforge.net/sox.html 300 // To remove the silence from both beginning and end of the audio file, we 301 // call sox silence command twice: once on normal file and again on its 302 // reverse, then we reverse the final output. 303 // Silence parameters are (in sequence): 304 // ABOVE_PERIODS: The period for which silence occurs. Value 1 is used for 305 // silence at beginning of audio. 306 // DURATION: the amount of time in seconds that non-silence must be detected 307 // before sox stops trimming audio. 308 // THRESHOLD: value used to indicate what sample value is treates as silence. 309 const char* kAbovePeriods = "1"; 310 const char* kDuration = "2"; 311 const char* kTreshold = "5%"; 312 313#if defined(OS_WIN) 314 CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append( 315 FILE_PATH_LITERAL("sox.exe"))); 316#else 317 CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("sox"))); 318#endif 319 command_line.AppendArgPath(input_file); 320 command_line.AppendArgPath(output_file); 321 command_line.AppendArg("silence"); 322 command_line.AppendArg(kAbovePeriods); 323 command_line.AppendArg(kDuration); 324 command_line.AppendArg(kTreshold); 325 command_line.AppendArg("reverse"); 326 command_line.AppendArg("silence"); 327 command_line.AppendArg(kAbovePeriods); 328 command_line.AppendArg(kDuration); 329 command_line.AppendArg(kTreshold); 330 command_line.AppendArg("reverse"); 331 332 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 333 std::string result; 334 bool ok = base::GetAppOutput(command_line, &result); 335 LOG(INFO) << "Output was:\n\n" << result; 336 return ok; 337} 338 339bool CanParseAsFloat(const std::string& value) { 340 return atof(value.c_str()) != 0 || value == "0"; 341} 342 343// Runs PESQ to compare |reference_file| to a |actual_file|. The |sample_rate| 344// can be either 16000 or 8000. 345// 346// PESQ is only mono-aware, so the files should preferably be recorded in mono. 347// Furthermore it expects the file to be 16 rather than 32 bits, even though 348// 32 bits might work. The audio bandwidth of the two files should be the same 349// e.g. don't compare a 32 kHz file to a 8 kHz file. 350// 351// The raw score in MOS is written to |raw_mos|, whereas the MOS-LQO score is 352// written to mos_lqo. The scores are returned as floats in string form (e.g. 353// "3.145", etc). Returns true on success. 354bool RunPesq(const base::FilePath& reference_file, 355 const base::FilePath& actual_file, 356 int sample_rate, std::string* raw_mos, std::string* mos_lqo) { 357 // PESQ will break if the paths are too long (!). 358 EXPECT_LT(reference_file.value().length(), 128u); 359 EXPECT_LT(actual_file.value().length(), 128u); 360 361#if defined(OS_WIN) 362 base::FilePath pesq_path = 363 GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq.exe")); 364#else 365 base::FilePath pesq_path = 366 GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq")); 367#endif 368 369 if (!base::PathExists(pesq_path)) { 370 LOG(ERROR) << "Missing PESQ binary in " << pesq_path.value(); 371 return false; 372 } 373 374 CommandLine command_line(pesq_path); 375 command_line.AppendArg(base::StringPrintf("+%d", sample_rate)); 376 command_line.AppendArgPath(reference_file); 377 command_line.AppendArgPath(actual_file); 378 379 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 380 std::string result; 381 if (!base::GetAppOutput(command_line, &result)) { 382 LOG(ERROR) << "Failed to run PESQ."; 383 return false; 384 } 385 LOG(INFO) << "Output was:\n\n" << result; 386 387 const std::string result_anchor = "Prediction (Raw MOS, MOS-LQO): = "; 388 std::size_t anchor_pos = result.find(result_anchor); 389 if (anchor_pos == std::string::npos) { 390 LOG(ERROR) << "PESQ was not able to compute a score; we probably recorded " 391 << "only silence."; 392 return false; 393 } 394 395 // There are two tab-separated numbers on the format x.xxx, e.g. 5 chars each. 396 std::size_t first_number_pos = anchor_pos + result_anchor.length(); 397 *raw_mos = result.substr(first_number_pos, 5); 398 EXPECT_TRUE(CanParseAsFloat(*raw_mos)) << "Failed to parse raw MOS number."; 399 *mos_lqo = result.substr(first_number_pos + 5 + 1, 5); 400 EXPECT_TRUE(CanParseAsFloat(*mos_lqo)) << "Failed to parse MOS LQO number."; 401 402 return true; 403} 404 405#if defined(OS_LINUX) || defined(OS_WIN) 406// Only implemented on Linux and Windows for now. 407#define MAYBE_MANUAL_TestAudioQuality MANUAL_TestAudioQuality 408#else 409#define MAYBE_MANUAL_TestAudioQuality DISABLED_MANUAL_TestAudioQuality 410#endif 411 412IN_PROC_BROWSER_TEST_F(WebrtcAudioQualityBrowserTest, 413 MAYBE_MANUAL_TestAudioQuality) { 414#if defined(OS_WIN) 415 if (base::win::GetVersion() < base::win::VERSION_VISTA) { 416 // It would take work to implement this on XP; not prioritized right now. 417 LOG(INFO) << "This test is not implemented for Windows XP."; 418 return; 419 } 420#endif 421 ASSERT_TRUE(HasAllRequiredResources()); 422 // TODO(phoglund): make this use embedded_test_server when that test server 423 // can handle files > ~400Kb. 424 ASSERT_TRUE(test_server()->Start()); 425 ASSERT_TRUE(peerconnection_server_.Start()); 426 427 ForceMicrophoneVolumeTo100Percent(); 428 429 ui_test_utils::NavigateToURL( 430 browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage)); 431 content::WebContents* left_tab = 432 browser()->tab_strip_model()->GetActiveWebContents(); 433 434 chrome::AddBlankTabAt(browser(), -1, true); 435 content::WebContents* right_tab = 436 browser()->tab_strip_model()->GetActiveWebContents(); 437 ui_test_utils::NavigateToURL( 438 browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage)); 439 440 ConnectToPeerConnectionServer("peer 1", left_tab); 441 ConnectToPeerConnectionServer("peer 2", right_tab); 442 443 EXPECT_EQ("ok-peerconnection-created", 444 ExecuteJavascript("preparePeerConnection()", left_tab)); 445 446 AddAudioFile(kReferenceFileRelativeUrl, left_tab); 447 448 EstablishCall(left_tab, right_tab); 449 450 // Note: the media flow isn't necessarily established on the connection just 451 // because the ready state is ok on both sides. We sleep a bit between call 452 // establishment and playing to avoid cutting of the beginning of the audio 453 // file. 454 SleepInJavascript(left_tab, 2000); 455 456 base::FilePath recording = CreateTemporaryWaveFile(); 457 458 // Note: the sound clip is about 10 seconds: record for 15 seconds to get some 459 // safety margins on each side. 460 AudioRecorder recorder; 461 static int kRecordingTimeSeconds = 15; 462 ASSERT_TRUE(recorder.StartRecording(kRecordingTimeSeconds, recording, true)); 463 464 PlayAudioFile(left_tab); 465 466 ASSERT_TRUE(recorder.WaitForRecordingToEnd()); 467 LOG(INFO) << "Done recording to " << recording.value() << std::endl; 468 469 AssertNoAsynchronousErrors(left_tab); 470 AssertNoAsynchronousErrors(right_tab); 471 472 HangUp(left_tab); 473 WaitUntilHangupVerified(left_tab); 474 WaitUntilHangupVerified(right_tab); 475 476 AssertNoAsynchronousErrors(left_tab); 477 AssertNoAsynchronousErrors(right_tab); 478 479 base::FilePath trimmed_recording = CreateTemporaryWaveFile(); 480 481 ASSERT_TRUE(RemoveSilence(recording, trimmed_recording)); 482 LOG(INFO) << "Trimmed silence: " << trimmed_recording.value() << std::endl; 483 484 std::string raw_mos; 485 std::string mos_lqo; 486 base::FilePath reference_file_in_test_dir = 487 GetTestDataDir().Append(kReferenceFile); 488 ASSERT_TRUE(RunPesq(reference_file_in_test_dir, trimmed_recording, 16000, 489 &raw_mos, &mos_lqo)); 490 491 perf_test::PrintResult("audio_pesq", "", "raw_mos", raw_mos, "score", true); 492 perf_test::PrintResult("audio_pesq", "", "mos_lqo", mos_lqo, "score", true); 493 494 EXPECT_TRUE(base::DeleteFile(recording, false)); 495 EXPECT_TRUE(base::DeleteFile(trimmed_recording, false)); 496 497 ASSERT_TRUE(peerconnection_server_.Stop()); 498} 499