1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/command_line.h"
6#include "base/file_util.h"
7#include "base/path_service.h"
8#include "base/process/launch.h"
9#include "base/strings/stringprintf.h"
10#include "chrome/browser/media/webrtc_browsertest_base.h"
11#include "chrome/browser/media/webrtc_browsertest_common.h"
12#include "chrome/browser/profiles/profile.h"
13#include "chrome/browser/ui/browser.h"
14#include "chrome/browser/ui/browser_tabstrip.h"
15#include "chrome/browser/ui/tabs/tab_strip_model.h"
16#include "chrome/common/chrome_paths.h"
17#include "chrome/common/chrome_switches.h"
18#include "chrome/test/base/ui_test_utils.h"
19#include "chrome/test/perf/perf_test.h"
20#include "chrome/test/ui/ui_test.h"
21#include "content/public/test/browser_test_utils.h"
22#include "net/test/embedded_test_server/embedded_test_server.h"
23
24static const base::FilePath::CharType kPeerConnectionServer[] =
25#if defined(OS_WIN)
26    FILE_PATH_LITERAL("peerconnection_server.exe");
27#else
28    FILE_PATH_LITERAL("peerconnection_server");
29#endif
30
31static const base::FilePath::CharType kMediaPath[] =
32    FILE_PATH_LITERAL("pyauto_private/webrtc/");
33static const base::FilePath::CharType kToolsPath[] =
34    FILE_PATH_LITERAL("pyauto_private/media/tools");
35static const base::FilePath::CharType kReferenceFile[] =
36#if defined (OS_WIN)
37    FILE_PATH_LITERAL("human-voice-win.wav");
38#else
39    FILE_PATH_LITERAL("human-voice-linux.wav");
40#endif
41
42static const char kMainWebrtcTestHtmlPage[] =
43    "files/webrtc/webrtc_audio_quality_test.html";
44
45base::FilePath GetTestDataDir() {
46  base::FilePath source_dir;
47  PathService::Get(chrome::DIR_TEST_DATA, &source_dir);
48  return source_dir;
49}
50
51// Test we can set up a WebRTC call and play audio through it.
52//
53// This test will only work on machines that have been configured to record
54// their own input.
55//
56// On Linux:
57// 1. # sudo apt-get install pavucontrol
58// 2. For the user who will run the test: # pavucontrol
59// 3. In a separate terminal, # arecord dummy
60// 4. In pavucontrol, go to the recording tab.
61// 5. For the ALSA plug-in [aplay]: ALSA Capture from, change from <x> to
62//    <Monitor of x>, where x is whatever your primary sound device is called.
63//    This test expects the device id to be render.monitor - if it's something
64//    else, the microphone level will not get forced to 100% appropriately.
65//    See ForceMicrophoneVolumeTo100% for more details. You can list the
66//    available monitor devices on your system by running the command
67//    pacmd list-sources | grep name | grep monitor.
68// 6. Try launching chrome as the target user on the target machine, try
69//    playing, say, a YouTube video, and record with # arecord -f dat tmp.dat.
70//    Verify the recording with aplay (should have recorded what you played
71//    from chrome).
72//
73// On Windows 7:
74// 1. Control panel > Sound > Manage audio devices.
75// 2. In the recording tab, right-click in an empty space in the pane with the
76//    devices. Tick 'show disabled devices'.
77// 3. You should see a 'stero mix' device - this is what your speakers output.
78//    Right click > Properties.
79// 4. In the Listen tab for the mix device, check the 'listen to this device'
80//    checkbox. Ensure the mix device is the default recording device.
81// 5. Launch chrome and try playing a video with sound. You should see
82//    in the volume meter for the mix device. Configure the mix device to have
83//    50 / 100 in level. Also go into the playback tab, right-click Speakers,
84//    and set that level to 50 / 100. Otherwise you will get distortion in
85//    the recording.
86class WebrtcAudioQualityBrowserTest : public WebRtcTestBase {
87 public:
88  WebrtcAudioQualityBrowserTest()
89      : peerconnection_server_(base::kNullProcessHandle) {}
90
91  virtual void SetUp() OVERRIDE {
92    RunPeerConnectionServer();
93    InProcessBrowserTest::SetUp();
94  }
95
96  virtual void TearDown() OVERRIDE {
97    ShutdownPeerConnectionServer();
98    InProcessBrowserTest::TearDown();
99  }
100
101  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
102    // TODO(phoglund): check that user actually has the requisite devices and
103    // print a nice message if not; otherwise the test just times out which can
104    // be confusing.
105    // This test expects real device handling and requires a real webcam / audio
106    // device; it will not work with fake devices.
107    EXPECT_FALSE(command_line->HasSwitch(
108        switches::kUseFakeDeviceForMediaStream));
109    EXPECT_FALSE(command_line->HasSwitch(
110        switches::kUseFakeUIForMediaStream));
111
112    // Ensure we have the stuff we need.
113    base::FilePath reference_file =
114        GetTestDataDir().Append(kMediaPath).Append(kReferenceFile);
115    EXPECT_TRUE(base::PathExists(reference_file))
116        << "Cannot find the reference file to be used for audio quality "
117        << "comparison: " << reference_file.value();
118  }
119
120  void AddAudioFile(const base::FilePath& input_file_relative,
121                    content::WebContents* tab_contents) {
122    EXPECT_EQ("ok-added", ExecuteJavascript(
123        base::StringPrintf("addAudioFile('%s')",
124                           input_file_relative.value().c_str()), tab_contents));
125  }
126
127  void PlayAudioFile(content::WebContents* tab_contents) {
128    EXPECT_EQ("ok-playing", ExecuteJavascript("playAudioFile()", tab_contents));
129  }
130
131  // Convenience method which executes the provided javascript in the context
132  // of the provided web contents and returns what it evaluated to.
133  std::string ExecuteJavascript(const std::string& javascript,
134                                content::WebContents* tab_contents) {
135    std::string result;
136    EXPECT_TRUE(content::ExecuteScriptAndExtractString(
137        tab_contents, javascript, &result));
138    return result;
139  }
140
141  // Ensures we didn't get any errors asynchronously (e.g. while no javascript
142  // call from this test was outstanding).
143  // TODO(phoglund): this becomes obsolete when we switch to communicating with
144  // the DOM message queue.
145  void AssertNoAsynchronousErrors(content::WebContents* tab_contents) {
146    EXPECT_EQ("ok-no-errors",
147              ExecuteJavascript("getAnyTestFailures()", tab_contents));
148  }
149
150  // The peer connection server lets our two tabs find each other and talk to
151  // each other (e.g. it is the application-specific "signaling solution").
152  void ConnectToPeerConnectionServer(const std::string peer_name,
153                                     content::WebContents* tab_contents) {
154    std::string javascript = base::StringPrintf(
155        "connect('http://localhost:8888', '%s');", peer_name.c_str());
156    EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents));
157  }
158
159  void EstablishCall(content::WebContents* from_tab,
160                     content::WebContents* to_tab) {
161    EXPECT_EQ("ok-negotiating",
162              ExecuteJavascript("negotiateCall()", from_tab));
163
164    // Ensure the call gets up on both sides.
165    EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
166                                 "active", from_tab));
167    EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
168                                 "active", to_tab));
169  }
170
171  void HangUp(content::WebContents* from_tab) {
172    EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
173  }
174
175  void WaitUntilHangupVerified(content::WebContents* tab_contents) {
176    EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
177                                 "no-peer-connection", tab_contents));
178  }
179
180  base::FilePath CreateTemporaryWaveFile() {
181    base::FilePath filename;
182    EXPECT_TRUE(file_util::CreateTemporaryFile(&filename));
183    base::FilePath wav_filename =
184        filename.AddExtension(FILE_PATH_LITERAL(".wav"));
185    EXPECT_TRUE(base::Move(filename, wav_filename));
186    return wav_filename;
187  }
188
189 private:
190  void RunPeerConnectionServer() {
191    base::FilePath peerconnection_server;
192    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &peerconnection_server));
193    peerconnection_server = peerconnection_server.Append(kPeerConnectionServer);
194
195    EXPECT_TRUE(base::PathExists(peerconnection_server)) <<
196        "Missing peerconnection_server. You must build "
197        "it so it ends up next to the browser test binary.";
198    EXPECT_TRUE(base::LaunchProcess(
199        CommandLine(peerconnection_server),
200        base::LaunchOptions(),
201        &peerconnection_server_)) << "Failed to launch peerconnection_server.";
202  }
203
204  void ShutdownPeerConnectionServer() {
205    EXPECT_TRUE(base::KillProcess(peerconnection_server_, 0, false)) <<
206        "Failed to shut down peerconnection_server!";
207  }
208
209  base::ProcessHandle peerconnection_server_;
210};
211
212class AudioRecorder {
213 public:
214  AudioRecorder(): recording_application_(base::kNullProcessHandle) {}
215  ~AudioRecorder() {}
216
217  void StartRecording(int duration_sec, const base::FilePath& output_file,
218                      bool mono) {
219    EXPECT_EQ(base::kNullProcessHandle, recording_application_)
220        << "Tried to record, but is already recording.";
221
222    CommandLine command_line(CommandLine::NO_PROGRAM);
223#if defined(OS_WIN)
224    NOTREACHED();  // TODO(phoglund): implement.
225#else
226    int num_channels = mono ? 1 : 2;
227    command_line.SetProgram(base::FilePath("arecord"));
228    command_line.AppendArg("-d");
229    command_line.AppendArg(base::StringPrintf("%d", duration_sec));
230    command_line.AppendArg("-f");
231    command_line.AppendArg("dat");
232    command_line.AppendArg("-c");
233    command_line.AppendArg(base::StringPrintf("%d", num_channels));
234    command_line.AppendArgPath(output_file);
235#endif
236
237    LOG(INFO) << "Running " << command_line.GetCommandLineString();
238    EXPECT_TRUE(base::LaunchProcess(
239        command_line,
240        base::LaunchOptions(),
241        &recording_application_)) << "Failed to launch recording application.";
242  }
243
244  void WaitForRecordingToEnd() {
245    int exit_code;
246    EXPECT_TRUE(base::WaitForExitCode(recording_application_, &exit_code)) <<
247        "Failed to wait for recording to end.";
248    EXPECT_EQ(0, exit_code);
249  }
250 private:
251  base::ProcessHandle recording_application_;
252};
253
254void ForceMicrophoneVolumeTo100Percent() {
255#if defined(OS_WIN)
256  NOTREACHED();  // TODO(phoglund): implement.
257#else
258  const std::string kRecordingDeviceId = "render.monitor";
259  const std::string kHundredPercentVolume = "65536";
260
261  CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("pacmd")));
262  command_line.AppendArg("set-source-volume");
263  command_line.AppendArg(kRecordingDeviceId);
264  command_line.AppendArg(kHundredPercentVolume);
265  LOG(INFO) << "Running " << command_line.GetCommandLineString();
266  std::string result;
267  if (!base::GetAppOutput(command_line, &result)) {
268    // It's hard to figure out for instance the default PA recording device name
269    // for different systems, so just warn here. Users will most often have a
270    // reasonable mic level on their systems.
271    LOG(WARNING) << "Failed to set mic volume to 100% on your system. " <<
272        "The test may fail or have results distorted; please ensure that " <<
273        "your mic level is 100% manually.";
274  }
275#endif
276}
277
278// Removes silence from beginning and end of the |input_audio_file| and writes
279// the result to the |output_audio_file|.
280void RemoveSilence(const base::FilePath& input_file,
281                   const base::FilePath& output_file) {
282  // SOX documentation for silence command: http://sox.sourceforge.net/sox.html
283  // To remove the silence from both beginning and end of the audio file, we
284  // call sox silence command twice: once on normal file and again on its
285  // reverse, then we reverse the final output.
286  // Silence parameters are (in sequence):
287  // ABOVE_PERIODS: The period for which silence occurs. Value 1 is used for
288  //                 silence at beginning of audio.
289  // DURATION: the amount of time in seconds that non-silence must be detected
290  //           before sox stops trimming audio.
291  // THRESHOLD: value used to indicate what sample value is treates as silence.
292  const char* kAbovePeriods = "1";
293  const char* kDuration = "2";
294  const char* kTreshold = "5%";
295
296  CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("sox")));
297  command_line.AppendArgPath(input_file);
298  command_line.AppendArgPath(output_file);
299  command_line.AppendArg("silence");
300  command_line.AppendArg(kAbovePeriods);
301  command_line.AppendArg(kDuration);
302  command_line.AppendArg(kTreshold);
303  command_line.AppendArg("reverse");
304  command_line.AppendArg("silence");
305  command_line.AppendArg(kAbovePeriods);
306  command_line.AppendArg(kDuration);
307  command_line.AppendArg(kTreshold);
308  command_line.AppendArg("reverse");
309
310  LOG(INFO) << "Running " << command_line.GetCommandLineString();
311  std::string result;
312  EXPECT_TRUE(base::GetAppOutput(command_line, &result))
313      << "Failed to launch sox.";
314  LOG(INFO) << "Output was:\n\n" << result;
315}
316
317bool CanParseAsFloat(const std::string& value) {
318  return atof(value.c_str()) != 0 || value == "0";
319}
320
321// Runs PESQ to compare |reference_file| to a |actual_file|. The |sample_rate|
322// can be either 16000 or 8000.
323//
324// PESQ is only mono-aware, so the files should preferably be recorded in mono.
325// Furthermore it expects the file to be 16 rather than 32 bits, even though
326// 32 bits might work. The audio bandwidth of the two files should be the same
327// e.g. don't compare a 32 kHz file to a 8 kHz file.
328//
329// The raw score in MOS is written to |raw_mos|, whereas the MOS-LQO score is
330// written to mos_lqo. The scores are returned as floats in string form (e.g.
331// "3.145", etc).
332void RunPesq(const base::FilePath& reference_file,
333             const base::FilePath& actual_file,
334             int sample_rate, std::string* raw_mos, std::string* mos_lqo) {
335  // PESQ will break if the paths are too long (!).
336  EXPECT_LT(reference_file.value().length(), 128u);
337  EXPECT_LT(actual_file.value().length(), 128u);
338
339  base::FilePath pesq_path =
340      GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq"));
341  CommandLine command_line(pesq_path);
342  command_line.AppendArg(base::StringPrintf("+%d", sample_rate));
343  command_line.AppendArgPath(reference_file);
344  command_line.AppendArgPath(actual_file);
345
346  LOG(INFO) << "Running " << command_line.GetCommandLineString();
347  std::string result;
348  EXPECT_TRUE(base::GetAppOutput(command_line, &result))
349      << "Failed to launch pesq.";
350  LOG(INFO) << "Output was:\n\n" << result;
351
352  const std::string result_anchor = "Prediction (Raw MOS, MOS-LQO):  = ";
353  std::size_t anchor_pos = result.find(result_anchor);
354  EXPECT_NE(std::string::npos, anchor_pos);
355
356  // There are two tab-separated numbers on the format x.xxx, e.g. 5 chars each.
357  std::size_t first_number_pos = anchor_pos + result_anchor.length();
358  *raw_mos = result.substr(first_number_pos, 5);
359  EXPECT_TRUE(CanParseAsFloat(*raw_mos)) << "Failed to parse raw MOS number.";
360  *mos_lqo = result.substr(first_number_pos + 5 + 1, 5);
361  EXPECT_TRUE(CanParseAsFloat(*mos_lqo)) << "Failed to parse MOS LQO number.";
362}
363
364#if defined(OS_LINUX)
365// Only implemented on Linux for now.
366#define MAYBE_MANUAL_TestAudioQuality MANUAL_TestAudioQuality
367#else
368#define MAYBE_MANUAL_TestAudioQuality DISABLED_MANUAL_TestAudioQuality
369#endif
370
371IN_PROC_BROWSER_TEST_F(WebrtcAudioQualityBrowserTest,
372                       MAYBE_MANUAL_TestAudioQuality) {
373  EXPECT_TRUE(test_server()->Start());
374
375  ForceMicrophoneVolumeTo100Percent();
376
377  ui_test_utils::NavigateToURL(
378      browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage));
379  content::WebContents* left_tab =
380      browser()->tab_strip_model()->GetActiveWebContents();
381
382  chrome::AddBlankTabAt(browser(), -1, true);
383  content::WebContents* right_tab =
384      browser()->tab_strip_model()->GetActiveWebContents();
385  ui_test_utils::NavigateToURL(
386        browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage));
387
388  ConnectToPeerConnectionServer("peer 1", left_tab);
389  ConnectToPeerConnectionServer("peer 2", right_tab);
390
391  EXPECT_EQ("ok-peerconnection-created",
392            ExecuteJavascript("preparePeerConnection()", left_tab));
393
394  base::FilePath reference_file =
395      base::FilePath(kMediaPath).Append(kReferenceFile);
396
397  // The javascript will load the reference file relative to its location,
398  // which is in /webrtc on the web server. Therefore, prepend a '..' traversal.
399  AddAudioFile(base::FilePath(FILE_PATH_LITERAL("..")).Append(reference_file),
400               left_tab);
401
402  EstablishCall(left_tab, right_tab);
403
404  // Note: the media flow isn't necessarily established on the connection just
405  // because the ready state is ok on both sides. We sleep a bit between call
406  // establishment and playing to avoid cutting of the beginning of the audio
407  // file.
408  SleepInJavascript(left_tab, 2000);
409
410  base::FilePath recording = CreateTemporaryWaveFile();
411
412  // Note: the sound clip is about 10 seconds: record for 15 seconds to get some
413  // safety margins on each side.
414  AudioRecorder recorder;
415  static int kRecordingTimeSeconds = 15;
416  recorder.StartRecording(kRecordingTimeSeconds, recording, true);
417
418  PlayAudioFile(left_tab);
419
420  recorder.WaitForRecordingToEnd();
421  LOG(INFO) << "Done recording to " << recording.value() << std::endl;
422
423  AssertNoAsynchronousErrors(left_tab);
424  AssertNoAsynchronousErrors(right_tab);
425
426  HangUp(left_tab);
427  WaitUntilHangupVerified(left_tab);
428  WaitUntilHangupVerified(right_tab);
429
430  AssertNoAsynchronousErrors(left_tab);
431  AssertNoAsynchronousErrors(right_tab);
432
433  base::FilePath trimmed_recording = CreateTemporaryWaveFile();
434
435  RemoveSilence(recording, trimmed_recording);
436  LOG(INFO) << "Trimmed silence: " << trimmed_recording.value() << std::endl;
437
438  std::string raw_mos;
439  std::string mos_lqo;
440  base::FilePath reference_file_in_test_dir =
441      GetTestDataDir().Append(reference_file);
442  RunPesq(reference_file_in_test_dir, trimmed_recording, 16000, &raw_mos,
443          &mos_lqo);
444
445  perf_test::PrintResult("audio_pesq", "", "raw_mos", raw_mos, "score", true);
446  perf_test::PrintResult("audio_pesq", "", "mos_lqo", mos_lqo, "score", true);
447
448  EXPECT_TRUE(base::DeleteFile(recording, false));
449  EXPECT_TRUE(base::DeleteFile(trimmed_recording, false));
450}
451