chrome_webrtc_audio_quality_browsertest.cc revision 8bcbed890bc3ce4d7a057a8f32cab53fa534672e
1// Copyright 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <ctime> 6 7#include "base/command_line.h" 8#include "base/file_util.h" 9#include "base/path_service.h" 10#include "base/process/launch.h" 11#include "base/scoped_native_library.h" 12#include "base/strings/stringprintf.h" 13#include "base/win/windows_version.h" 14#include "chrome/browser/media/webrtc_browsertest_base.h" 15#include "chrome/browser/media/webrtc_browsertest_common.h" 16#include "chrome/browser/profiles/profile.h" 17#include "chrome/browser/ui/browser.h" 18#include "chrome/browser/ui/browser_tabstrip.h" 19#include "chrome/browser/ui/tabs/tab_strip_model.h" 20#include "chrome/common/chrome_paths.h" 21#include "chrome/common/chrome_switches.h" 22#include "chrome/test/base/ui_test_utils.h" 23#include "chrome/test/ui/ui_test.h" 24#include "content/public/test/browser_test_utils.h" 25#include "net/test/embedded_test_server/embedded_test_server.h" 26#include "testing/perf/perf_test.h" 27 28static const base::FilePath::CharType kReferenceFile[] = 29#if defined (OS_WIN) 30 FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-win.wav"); 31#else 32 FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-linux.wav"); 33#endif 34 35// The javascript will load the reference file relative to its location, 36// which is in /webrtc on the web server. Therefore, prepend a '..' traversal. 37static const char kReferenceFileRelativeUrl[] = 38#if defined (OS_WIN) 39 "../pyauto_private/webrtc/human-voice-win.wav"; 40#else 41 "../pyauto_private/webrtc/human-voice-linux.wav"; 42#endif 43 44static const base::FilePath::CharType kToolsPath[] = 45 FILE_PATH_LITERAL("pyauto_private/media/tools"); 46 47static const char kMainWebrtcTestHtmlPage[] = 48 "files/webrtc/webrtc_audio_quality_test.html"; 49 50static base::FilePath GetTestDataDir() { 51 base::FilePath source_dir; 52 PathService::Get(chrome::DIR_TEST_DATA, &source_dir); 53 return source_dir; 54} 55 56// Test we can set up a WebRTC call and play audio through it. 57// 58// You must have the src-internal solution in your .gclient to put the required 59// pyauto_private directory into chrome/test/data/. 60// 61// This test will only work on machines that have been configured to record 62// their own input. 63// 64// On Linux: 65// 1. # sudo apt-get install pavucontrol 66// 2. For the user who will run the test: # pavucontrol 67// 3. In a separate terminal, # arecord dummy 68// 4. In pavucontrol, go to the recording tab. 69// 5. For the ALSA plug-in [aplay]: ALSA Capture from, change from <x> to 70// <Monitor of x>, where x is whatever your primary sound device is called. 71// 6. Try launching chrome as the target user on the target machine, try 72// playing, say, a YouTube video, and record with # arecord -f dat tmp.dat. 73// Verify the recording with aplay (should have recorded what you played 74// from chrome). 75// 76// Note: the volume for ALL your input devices will be forced to 100% by 77// running this test on Linux. 78// 79// On Windows 7: 80// 1. Control panel > Sound > Manage audio devices. 81// 2. In the recording tab, right-click in an empty space in the pane with the 82// devices. Tick 'show disabled devices'. 83// 3. You should see a 'stero mix' device - this is what your speakers output. 84// Right click > Properties. 85// 4. In the Listen tab for the mix device, check the 'listen to this device' 86// checkbox. Ensure the mix device is the default recording device. 87// 5. Launch chrome and try playing a video with sound. You should see 88// in the volume meter for the mix device. Configure the mix device to have 89// 50 / 100 in level. Also go into the playback tab, right-click Speakers, 90// and set that level to 50 / 100. Otherwise you will get distortion in 91// the recording. 92class WebrtcAudioQualityBrowserTest : public WebRtcTestBase { 93 public: 94 virtual void SetUpInProcessBrowserTestFixture() OVERRIDE { 95 PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem(); 96 } 97 98 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 99 // This test expects real device handling and requires a real webcam / audio 100 // device; it will not work with fake devices. 101 EXPECT_FALSE(command_line->HasSwitch( 102 switches::kUseFakeDeviceForMediaStream)); 103 EXPECT_FALSE(command_line->HasSwitch( 104 switches::kUseFakeUIForMediaStream)); 105 } 106 107 bool HasAllRequiredResources() { 108 base::FilePath reference_file = 109 GetTestDataDir().Append(kReferenceFile); 110 if (!base::PathExists(reference_file)) { 111 LOG(ERROR) << "Cannot find the reference file to be used for audio " 112 << "quality comparison: " << reference_file.value(); 113 return false; 114 } 115 return true; 116 } 117 118 void AddAudioFile(const std::string& input_file_relative_url, 119 content::WebContents* tab_contents) { 120 EXPECT_EQ("ok-added", ExecuteJavascript( 121 "addAudioFile('" + input_file_relative_url + "')", tab_contents)); 122 } 123 124 void PlayAudioFile(content::WebContents* tab_contents) { 125 EXPECT_EQ("ok-playing", ExecuteJavascript("playAudioFile()", tab_contents)); 126 } 127 128 // Ensures we didn't get any errors asynchronously (e.g. while no javascript 129 // call from this test was outstanding). 130 // TODO(phoglund): this becomes obsolete when we switch to communicating with 131 // the DOM message queue. 132 void AssertNoAsynchronousErrors(content::WebContents* tab_contents) { 133 EXPECT_EQ("ok-no-errors", 134 ExecuteJavascript("getAnyTestFailures()", tab_contents)); 135 } 136 137 void EstablishCall(content::WebContents* from_tab, 138 content::WebContents* to_tab) { 139 EXPECT_EQ("ok-negotiating", 140 ExecuteJavascript("negotiateCall()", from_tab)); 141 142 // Ensure the call gets up on both sides. 143 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 144 "active", from_tab)); 145 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 146 "active", to_tab)); 147 } 148 149 void HangUp(content::WebContents* from_tab) { 150 EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab)); 151 } 152 153 void WaitUntilHangupVerified(content::WebContents* tab_contents) { 154 EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()", 155 "no-peer-connection", tab_contents)); 156 } 157 158 base::FilePath CreateTemporaryWaveFile() { 159 base::FilePath filename; 160 EXPECT_TRUE(file_util::CreateTemporaryFile(&filename)); 161 base::FilePath wav_filename = 162 filename.AddExtension(FILE_PATH_LITERAL(".wav")); 163 EXPECT_TRUE(base::Move(filename, wav_filename)); 164 return wav_filename; 165 } 166 167 PeerConnectionServerRunner peerconnection_server_; 168}; 169 170class AudioRecorder { 171 public: 172 AudioRecorder(): recording_application_(base::kNullProcessHandle) {} 173 ~AudioRecorder() {} 174 175 // Starts the recording program for the specified duration. Returns true 176 // on success. 177 bool StartRecording(int duration_sec, const base::FilePath& output_file, 178 bool mono) { 179 EXPECT_EQ(base::kNullProcessHandle, recording_application_) 180 << "Tried to record, but is already recording."; 181 182 CommandLine command_line(CommandLine::NO_PROGRAM); 183#if defined(OS_WIN) 184 // This disable is required to run SoundRecorder.exe on 64-bit Windows 185 // from a 32-bit binary. We need to load the wow64 disable function from 186 // the DLL since it doesn't exist on Windows XP. 187 // TODO(phoglund): find some cleaner solution than using SoundRecorder.exe. 188 base::ScopedNativeLibrary kernel32_lib(base::FilePath(L"kernel32")); 189 if (kernel32_lib.is_valid()) { 190 typedef BOOL (WINAPI* Wow64DisableWow64FSRedirection)(PVOID*); 191 Wow64DisableWow64FSRedirection wow_64_disable_wow_64_fs_redirection; 192 wow_64_disable_wow_64_fs_redirection = 193 reinterpret_cast<Wow64DisableWow64FSRedirection>( 194 kernel32_lib.GetFunctionPointer( 195 "Wow64DisableWow64FsRedirection")); 196 if (wow_64_disable_wow_64_fs_redirection != NULL) { 197 PVOID* ignored = NULL; 198 wow_64_disable_wow_64_fs_redirection(ignored); 199 } 200 } 201 202 char duration_in_hms[128] = {0}; 203 struct tm duration_tm = {0}; 204 duration_tm.tm_sec = duration_sec; 205 EXPECT_NE(0u, strftime(duration_in_hms, arraysize(duration_in_hms), 206 "%H:%M:%S", &duration_tm)); 207 208 command_line.SetProgram( 209 base::FilePath(FILE_PATH_LITERAL("SoundRecorder.exe"))); 210 command_line.AppendArg("/FILE"); 211 command_line.AppendArgPath(output_file); 212 command_line.AppendArg("/DURATION"); 213 command_line.AppendArg(duration_in_hms); 214#else 215 int num_channels = mono ? 1 : 2; 216 command_line.SetProgram(base::FilePath("arecord")); 217 command_line.AppendArg("-d"); 218 command_line.AppendArg(base::StringPrintf("%d", duration_sec)); 219 command_line.AppendArg("-f"); 220 command_line.AppendArg("dat"); 221 command_line.AppendArg("-c"); 222 command_line.AppendArg(base::StringPrintf("%d", num_channels)); 223 command_line.AppendArgPath(output_file); 224#endif 225 226 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 227 return base::LaunchProcess(command_line, base::LaunchOptions(), 228 &recording_application_); 229 } 230 231 // Joins the recording program. Returns true on success. 232 bool WaitForRecordingToEnd() { 233 int exit_code = -1; 234 base::WaitForExitCode(recording_application_, &exit_code); 235 return exit_code == 0; 236 } 237 private: 238 base::ProcessHandle recording_application_; 239}; 240 241bool ForceMicrophoneVolumeTo100Percent() { 242#if defined(OS_WIN) 243 CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append( 244 FILE_PATH_LITERAL("force_mic_volume_max.exe"))); 245 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 246 std::string result; 247 if (!base::GetAppOutput(command_line, &result)) { 248 LOG(ERROR) << "Failed to set source volume: output was " << result; 249 return false; 250 } 251#else 252 // Just force the volume of, say the first 5 devices. A machine will rarely 253 // have more input sources than that. This is way easier than finding the 254 // input device we happen to be using. 255 for (int device_index = 0; device_index < 5; ++device_index) { 256 std::string result; 257 const std::string kHundredPercentVolume = "65536"; 258 CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("pacmd"))); 259 command_line.AppendArg("set-source-volume"); 260 command_line.AppendArg(base::StringPrintf("%d", device_index)); 261 command_line.AppendArg(kHundredPercentVolume); 262 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 263 if (!base::GetAppOutput(command_line, &result)) { 264 LOG(ERROR) << "Failed to set source volume: output was " << result; 265 return false; 266 } 267 } 268#endif 269 return true; 270} 271 272// Removes silence from beginning and end of the |input_audio_file| and writes 273// the result to the |output_audio_file|. Returns true on success. 274bool RemoveSilence(const base::FilePath& input_file, 275 const base::FilePath& output_file) { 276 // SOX documentation for silence command: http://sox.sourceforge.net/sox.html 277 // To remove the silence from both beginning and end of the audio file, we 278 // call sox silence command twice: once on normal file and again on its 279 // reverse, then we reverse the final output. 280 // Silence parameters are (in sequence): 281 // ABOVE_PERIODS: The period for which silence occurs. Value 1 is used for 282 // silence at beginning of audio. 283 // DURATION: the amount of time in seconds that non-silence must be detected 284 // before sox stops trimming audio. 285 // THRESHOLD: value used to indicate what sample value is treates as silence. 286 const char* kAbovePeriods = "1"; 287 const char* kDuration = "2"; 288 const char* kTreshold = "5%"; 289 290#if defined(OS_WIN) 291 CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append( 292 FILE_PATH_LITERAL("sox.exe"))); 293#else 294 CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("sox"))); 295#endif 296 command_line.AppendArgPath(input_file); 297 command_line.AppendArgPath(output_file); 298 command_line.AppendArg("silence"); 299 command_line.AppendArg(kAbovePeriods); 300 command_line.AppendArg(kDuration); 301 command_line.AppendArg(kTreshold); 302 command_line.AppendArg("reverse"); 303 command_line.AppendArg("silence"); 304 command_line.AppendArg(kAbovePeriods); 305 command_line.AppendArg(kDuration); 306 command_line.AppendArg(kTreshold); 307 command_line.AppendArg("reverse"); 308 309 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 310 std::string result; 311 bool ok = base::GetAppOutput(command_line, &result); 312 LOG(INFO) << "Output was:\n\n" << result; 313 return ok; 314} 315 316bool CanParseAsFloat(const std::string& value) { 317 return atof(value.c_str()) != 0 || value == "0"; 318} 319 320// Runs PESQ to compare |reference_file| to a |actual_file|. The |sample_rate| 321// can be either 16000 or 8000. 322// 323// PESQ is only mono-aware, so the files should preferably be recorded in mono. 324// Furthermore it expects the file to be 16 rather than 32 bits, even though 325// 32 bits might work. The audio bandwidth of the two files should be the same 326// e.g. don't compare a 32 kHz file to a 8 kHz file. 327// 328// The raw score in MOS is written to |raw_mos|, whereas the MOS-LQO score is 329// written to mos_lqo. The scores are returned as floats in string form (e.g. 330// "3.145", etc). Returns true on success. 331bool RunPesq(const base::FilePath& reference_file, 332 const base::FilePath& actual_file, 333 int sample_rate, std::string* raw_mos, std::string* mos_lqo) { 334 // PESQ will break if the paths are too long (!). 335 EXPECT_LT(reference_file.value().length(), 128u); 336 EXPECT_LT(actual_file.value().length(), 128u); 337 338#if defined(OS_WIN) 339 base::FilePath pesq_path = 340 GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq.exe")); 341#else 342 base::FilePath pesq_path = 343 GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq")); 344#endif 345 346 if (!base::PathExists(pesq_path)) { 347 LOG(ERROR) << "Missing PESQ binary in " << pesq_path.value(); 348 return false; 349 } 350 351 CommandLine command_line(pesq_path); 352 command_line.AppendArg(base::StringPrintf("+%d", sample_rate)); 353 command_line.AppendArgPath(reference_file); 354 command_line.AppendArgPath(actual_file); 355 356 LOG(INFO) << "Running " << command_line.GetCommandLineString(); 357 std::string result; 358 if (!base::GetAppOutput(command_line, &result)) { 359 LOG(ERROR) << "Failed to run PESQ."; 360 return false; 361 } 362 LOG(INFO) << "Output was:\n\n" << result; 363 364 const std::string result_anchor = "Prediction (Raw MOS, MOS-LQO): = "; 365 std::size_t anchor_pos = result.find(result_anchor); 366 if (anchor_pos == std::string::npos) { 367 LOG(ERROR) << "PESQ was not able to compute a score; we probably recorded " 368 << "only silence."; 369 return false; 370 } 371 372 // There are two tab-separated numbers on the format x.xxx, e.g. 5 chars each. 373 std::size_t first_number_pos = anchor_pos + result_anchor.length(); 374 *raw_mos = result.substr(first_number_pos, 5); 375 EXPECT_TRUE(CanParseAsFloat(*raw_mos)) << "Failed to parse raw MOS number."; 376 *mos_lqo = result.substr(first_number_pos + 5 + 1, 5); 377 EXPECT_TRUE(CanParseAsFloat(*mos_lqo)) << "Failed to parse MOS LQO number."; 378 379 return true; 380} 381 382#if defined(OS_LINUX) || defined(OS_WIN) 383// Only implemented on Linux and Windows for now. 384#define MAYBE_MANUAL_TestAudioQuality MANUAL_TestAudioQuality 385#else 386#define MAYBE_MANUAL_TestAudioQuality DISABLED_MANUAL_TestAudioQuality 387#endif 388 389IN_PROC_BROWSER_TEST_F(WebrtcAudioQualityBrowserTest, 390 MAYBE_MANUAL_TestAudioQuality) { 391#if defined(OS_WIN) 392 if (base::win::GetVersion() < base::win::VERSION_VISTA) { 393 // It would take work to implement this on XP; not prioritized right now. 394 LOG(ERROR) << "This test is not implemented for Windows XP."; 395 return; 396 } 397#endif 398 ASSERT_TRUE(HasAllRequiredResources()); 399 // TODO(phoglund): make this use embedded_test_server when that test server 400 // can handle files > ~400Kb. 401 ASSERT_TRUE(test_server()->Start()); 402 ASSERT_TRUE(peerconnection_server_.Start()); 403 404 ASSERT_TRUE(ForceMicrophoneVolumeTo100Percent()); 405 406 ui_test_utils::NavigateToURL( 407 browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage)); 408 content::WebContents* left_tab = 409 browser()->tab_strip_model()->GetActiveWebContents(); 410 411 chrome::AddBlankTabAt(browser(), -1, true); 412 content::WebContents* right_tab = 413 browser()->tab_strip_model()->GetActiveWebContents(); 414 ui_test_utils::NavigateToURL( 415 browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage)); 416 417 ConnectToPeerConnectionServer("peer 1", left_tab); 418 ConnectToPeerConnectionServer("peer 2", right_tab); 419 420 EXPECT_EQ("ok-peerconnection-created", 421 ExecuteJavascript("preparePeerConnection(false, true)", left_tab)); 422 423 AddAudioFile(kReferenceFileRelativeUrl, left_tab); 424 425 EstablishCall(left_tab, right_tab); 426 427 // Note: the media flow isn't necessarily established on the connection just 428 // because the ready state is ok on both sides. We sleep a bit between call 429 // establishment and playing to avoid cutting of the beginning of the audio 430 // file. 431 SleepInJavascript(left_tab, 2000); 432 433 base::FilePath recording = CreateTemporaryWaveFile(); 434 435 // Note: the sound clip is about 10 seconds: record for 15 seconds to get some 436 // safety margins on each side. 437 AudioRecorder recorder; 438 static int kRecordingTimeSeconds = 15; 439 ASSERT_TRUE(recorder.StartRecording(kRecordingTimeSeconds, recording, true)); 440 441 PlayAudioFile(left_tab); 442 443 ASSERT_TRUE(recorder.WaitForRecordingToEnd()); 444 LOG(INFO) << "Done recording to " << recording.value() << std::endl; 445 446 AssertNoAsynchronousErrors(left_tab); 447 AssertNoAsynchronousErrors(right_tab); 448 449 HangUp(left_tab); 450 WaitUntilHangupVerified(left_tab); 451 WaitUntilHangupVerified(right_tab); 452 453 AssertNoAsynchronousErrors(left_tab); 454 AssertNoAsynchronousErrors(right_tab); 455 456 base::FilePath trimmed_recording = CreateTemporaryWaveFile(); 457 458 ASSERT_TRUE(RemoveSilence(recording, trimmed_recording)); 459 LOG(INFO) << "Trimmed silence: " << trimmed_recording.value() << std::endl; 460 461 std::string raw_mos; 462 std::string mos_lqo; 463 base::FilePath reference_file_in_test_dir = 464 GetTestDataDir().Append(kReferenceFile); 465 ASSERT_TRUE(RunPesq(reference_file_in_test_dir, trimmed_recording, 16000, 466 &raw_mos, &mos_lqo)); 467 468 perf_test::PrintResult("audio_pesq", "", "raw_mos", raw_mos, "score", true); 469 perf_test::PrintResult("audio_pesq", "", "mos_lqo", mos_lqo, "score", true); 470 471 EXPECT_TRUE(base::DeleteFile(recording, false)); 472 EXPECT_TRUE(base::DeleteFile(trimmed_recording, false)); 473 474 ASSERT_TRUE(peerconnection_server_.Stop()); 475} 476