1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/renderer/media/webrtc_audio_device_impl.h"
6
7#include "base/bind.h"
8#include "base/metrics/histogram.h"
9#include "base/strings/string_util.h"
10#include "base/win/windows_version.h"
11#include "content/renderer/media/media_stream_audio_processor.h"
12#include "content/renderer/media/webrtc_audio_capturer.h"
13#include "content/renderer/media/webrtc_audio_renderer.h"
14#include "content/renderer/render_thread_impl.h"
15#include "media/audio/audio_parameters.h"
16#include "media/audio/sample_rates.h"
17
18using media::AudioParameters;
19using media::ChannelLayout;
20
21namespace content {
22
23WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
24    : ref_count_(0),
25      audio_transport_callback_(NULL),
26      input_delay_ms_(0),
27      output_delay_ms_(0),
28      initialized_(false),
29      playing_(false),
30      recording_(false),
31      microphone_volume_(0),
32      is_audio_track_processing_enabled_(
33          MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled()) {
34  DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
35}
36
37WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
38  DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
39  DCHECK(thread_checker_.CalledOnValidThread());
40  Terminate();
41}
42
43int32_t WebRtcAudioDeviceImpl::AddRef() {
44  DCHECK(thread_checker_.CalledOnValidThread());
45  return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
46}
47
48int32_t WebRtcAudioDeviceImpl::Release() {
49  DCHECK(thread_checker_.CalledOnValidThread());
50  int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
51  if (ret == 0) {
52    delete this;
53  }
54  return ret;
55}
56int WebRtcAudioDeviceImpl::OnData(const int16* audio_data,
57                                  int sample_rate,
58                                  int number_of_channels,
59                                  int number_of_frames,
60                                  const std::vector<int>& channels,
61                                  int audio_delay_milliseconds,
62                                  int current_volume,
63                                  bool need_audio_processing,
64                                  bool key_pressed) {
65  int total_delay_ms = 0;
66  {
67    base::AutoLock auto_lock(lock_);
68    // Return immediately when not recording or |channels| is empty.
69    // See crbug.com/274017: renderer crash dereferencing invalid channels[0].
70    if (!recording_ || channels.empty())
71      return 0;
72
73    // Store the reported audio delay locally.
74    input_delay_ms_ = audio_delay_milliseconds;
75    total_delay_ms = input_delay_ms_ + output_delay_ms_;
76    DVLOG(2) << "total delay: " << input_delay_ms_ + output_delay_ms_;
77  }
78
79  // Write audio frames in blocks of 10 milliseconds to the registered
80  // webrtc::AudioTransport sink. Keep writing until our internal byte
81  // buffer is empty.
82  const int16* audio_buffer = audio_data;
83  const int frames_per_10_ms = (sample_rate / 100);
84  CHECK_EQ(number_of_frames % frames_per_10_ms, 0);
85  int accumulated_audio_frames = 0;
86  uint32_t new_volume = 0;
87
88  // The lock here is to protect a race in the resampler inside webrtc when
89  // there are more than one input stream calling OnData(), which can happen
90  // when the users setup two getUserMedia, one for the microphone, another
91  // for WebAudio. Currently we don't have a better way to fix it except for
92  // adding a lock here to sequence the call.
93  // TODO(xians): Remove this workaround after we move the
94  // webrtc::AudioProcessing module to Chrome. See http://crbug/264611 for
95  // details.
96  base::AutoLock auto_lock(capture_callback_lock_);
97  while (accumulated_audio_frames < number_of_frames) {
98    // Deliver 10ms of recorded 16-bit linear PCM audio.
99    int new_mic_level = audio_transport_callback_->OnDataAvailable(
100        &channels[0],
101        channels.size(),
102        audio_buffer,
103        sample_rate,
104        number_of_channels,
105        frames_per_10_ms,
106        total_delay_ms,
107        current_volume,
108        key_pressed,
109        need_audio_processing);
110
111    accumulated_audio_frames += frames_per_10_ms;
112    audio_buffer += frames_per_10_ms * number_of_channels;
113
114    // The latest non-zero new microphone level will be returned.
115    if (new_mic_level)
116      new_volume = new_mic_level;
117  }
118
119  return new_volume;
120}
121
122void WebRtcAudioDeviceImpl::OnSetFormat(
123    const media::AudioParameters& params) {
124  DVLOG(1) << "WebRtcAudioDeviceImpl::OnSetFormat()";
125}
126
127void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
128                                       int sample_rate,
129                                       int audio_delay_milliseconds,
130                                       base::TimeDelta* current_time) {
131  render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
132
133  {
134    base::AutoLock auto_lock(lock_);
135    DCHECK(audio_transport_callback_);
136    // Store the reported audio delay locally.
137    output_delay_ms_ = audio_delay_milliseconds;
138  }
139
140  int frames_per_10_ms = (sample_rate / 100);
141  int bytes_per_sample = sizeof(render_buffer_[0]);
142  const int bytes_per_10_ms =
143      audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
144  DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
145
146  // Get audio frames in blocks of 10 milliseconds from the registered
147  // webrtc::AudioTransport source. Keep reading until our internal buffer
148  // is full.
149  uint32_t num_audio_frames = 0;
150  int accumulated_audio_frames = 0;
151  int16* audio_data = &render_buffer_[0];
152  while (accumulated_audio_frames < audio_bus->frames()) {
153    // Get 10ms and append output to temporary byte buffer.
154    int64_t elapsed_time_ms = -1;
155    int64_t ntp_time_ms = -1;
156    if (is_audio_track_processing_enabled_) {
157      // When audio processing is enabled in the audio track, we use
158      // PullRenderData() instead of NeedMorePlayData() to avoid passing the
159      // render data to the APM in WebRTC as reference signal for echo
160      // cancellation.
161      static const int kBitsPerByte = 8;
162      audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
163                                                sample_rate,
164                                                audio_bus->channels(),
165                                                frames_per_10_ms,
166                                                audio_data,
167                                                &elapsed_time_ms,
168                                                &ntp_time_ms);
169      accumulated_audio_frames += frames_per_10_ms;
170    } else {
171      // TODO(xians): Remove the following code after the APM in WebRTC is
172      // deprecated.
173      audio_transport_callback_->NeedMorePlayData(frames_per_10_ms,
174                                                  bytes_per_sample,
175                                                  audio_bus->channels(),
176                                                  sample_rate,
177                                                  audio_data,
178                                                  num_audio_frames,
179                                                  &elapsed_time_ms,
180                                                  &ntp_time_ms);
181      accumulated_audio_frames += num_audio_frames;
182    }
183    if (elapsed_time_ms >= 0) {
184      *current_time = base::TimeDelta::FromMilliseconds(elapsed_time_ms);
185    }
186    audio_data += bytes_per_10_ms;
187  }
188
189  // De-interleave each channel and convert to 32-bit floating-point
190  // with nominal range -1.0 -> +1.0 to match the callback format.
191  audio_bus->FromInterleaved(&render_buffer_[0],
192                             audio_bus->frames(),
193                             bytes_per_sample);
194
195  // Pass the render data to the playout sinks.
196  base::AutoLock auto_lock(lock_);
197  for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
198       it != playout_sinks_.end(); ++it) {
199    (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
200  }
201}
202
203void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
204  DCHECK(thread_checker_.CalledOnValidThread());
205  DCHECK_EQ(renderer, renderer_.get());
206  base::AutoLock auto_lock(lock_);
207  // Notify the playout sink of the change.
208  for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
209       it != playout_sinks_.end(); ++it) {
210    (*it)->OnPlayoutDataSourceChanged();
211  }
212
213  renderer_ = NULL;
214  playing_ = false;
215}
216
217int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
218    webrtc::AudioTransport* audio_callback) {
219  DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
220  DCHECK(thread_checker_.CalledOnValidThread());
221  DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
222  audio_transport_callback_ = audio_callback;
223  return 0;
224}
225
226int32_t WebRtcAudioDeviceImpl::Init() {
227  DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
228  DCHECK(thread_checker_.CalledOnValidThread());
229
230  // We need to return a success to continue the initialization of WebRtc VoE
231  // because failure on the capturer_ initialization should not prevent WebRTC
232  // from working. See issue http://crbug.com/144421 for details.
233  initialized_ = true;
234
235  return 0;
236}
237
238int32_t WebRtcAudioDeviceImpl::Terminate() {
239  DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
240  DCHECK(thread_checker_.CalledOnValidThread());
241
242  // Calling Terminate() multiple times in a row is OK.
243  if (!initialized_)
244    return 0;
245
246  StopRecording();
247  StopPlayout();
248
249  DCHECK(!renderer_.get() || !renderer_->IsStarted())
250      << "The shared audio renderer shouldn't be running";
251
252  // Stop all the capturers to ensure no further OnData() and
253  // RemoveAudioCapturer() callback.
254  // Cache the capturers in a local list since WebRtcAudioCapturer::Stop()
255  // will trigger RemoveAudioCapturer() callback.
256  CapturerList capturers;
257  capturers.swap(capturers_);
258  for (CapturerList::const_iterator iter = capturers.begin();
259       iter != capturers.end(); ++iter) {
260    (*iter)->Stop();
261  }
262
263  initialized_ = false;
264  return 0;
265}
266
267bool WebRtcAudioDeviceImpl::Initialized() const {
268  return initialized_;
269}
270
271int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
272  *available = initialized_;
273  return 0;
274}
275
276bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
277  return initialized_;
278}
279
280int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
281  *available = (!capturers_.empty());
282  return 0;
283}
284
285bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
286  DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
287  DCHECK(thread_checker_.CalledOnValidThread());
288  return (!capturers_.empty());
289}
290
291int32_t WebRtcAudioDeviceImpl::StartPlayout() {
292  DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
293  LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
294  {
295    base::AutoLock auto_lock(lock_);
296    if (!audio_transport_callback_)
297      return 0;
298  }
299
300  if (playing_) {
301    // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
302    // that the call is ignored the second time.
303    return 0;
304  }
305
306  playing_ = true;
307  return 0;
308}
309
310int32_t WebRtcAudioDeviceImpl::StopPlayout() {
311  DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
312  if (!playing_) {
313    // webrtc::VoiceEngine assumes that it is OK to call Stop() just in case.
314    return 0;
315  }
316
317  playing_ = false;
318  return 0;
319}
320
321bool WebRtcAudioDeviceImpl::Playing() const {
322  return playing_;
323}
324
325int32_t WebRtcAudioDeviceImpl::StartRecording() {
326  DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
327  DCHECK(initialized_);
328  LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
329  if (!audio_transport_callback_) {
330    return -1;
331  }
332
333  {
334    base::AutoLock auto_lock(lock_);
335    if (recording_)
336      return 0;
337
338    recording_ = true;
339  }
340
341  return 0;
342}
343
344int32_t WebRtcAudioDeviceImpl::StopRecording() {
345  DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
346  {
347    base::AutoLock auto_lock(lock_);
348    if (!recording_)
349      return 0;
350
351    recording_ = false;
352  }
353
354  return 0;
355}
356
357bool WebRtcAudioDeviceImpl::Recording() const {
358  base::AutoLock auto_lock(lock_);
359  return recording_;
360}
361
362int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
363  DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
364  DCHECK(initialized_);
365
366  // Only one microphone is supported at the moment, which is represented by
367  // the default capturer.
368  scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
369  if (!capturer.get())
370    return -1;
371
372  capturer->SetVolume(volume);
373  return 0;
374}
375
376// TODO(henrika): sort out calling thread once we start using this API.
377int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
378  DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
379  // We only support one microphone now, which is accessed via the default
380  // capturer.
381  DCHECK(initialized_);
382  scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
383  if (!capturer.get())
384    return -1;
385
386  *volume = static_cast<uint32_t>(capturer->Volume());
387
388  return 0;
389}
390
391int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
392  DCHECK(initialized_);
393  *max_volume = kMaxVolumeLevel;
394  return 0;
395}
396
397int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
398  *min_volume = 0;
399  return 0;
400}
401
402int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
403  DCHECK(initialized_);
404  *available = renderer_.get() && renderer_->channels() == 2;
405  return 0;
406}
407
408int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
409    bool* available) const {
410  DCHECK(initialized_);
411  // TODO(xians): These kind of hardware methods do not make much sense since we
412  // support multiple sources. Remove or figure out new APIs for such methods.
413  scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
414  if (!capturer.get())
415    return -1;
416
417  *available = (capturer->source_audio_parameters().channels() == 2);
418  return 0;
419}
420
421int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
422  base::AutoLock auto_lock(lock_);
423  *delay_ms = static_cast<uint16_t>(output_delay_ms_);
424  return 0;
425}
426
427int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
428  base::AutoLock auto_lock(lock_);
429  *delay_ms = static_cast<uint16_t>(input_delay_ms_);
430  return 0;
431}
432
433int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
434    uint32_t* sample_rate) const {
435  // We use the default capturer as the recording sample rate.
436  scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
437  if (!capturer.get())
438    return -1;
439
440  *sample_rate = static_cast<uint32_t>(
441      capturer->source_audio_parameters().sample_rate());
442  return 0;
443}
444
445int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
446    uint32_t* sample_rate) const {
447  *sample_rate = renderer_.get() ? renderer_->sample_rate() : 0;
448  return 0;
449}
450
451bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
452  DCHECK(thread_checker_.CalledOnValidThread());
453  DCHECK(renderer);
454
455  base::AutoLock auto_lock(lock_);
456  if (renderer_.get())
457    return false;
458
459  if (!renderer->Initialize(this))
460    return false;
461
462  renderer_ = renderer;
463  return true;
464}
465
466void WebRtcAudioDeviceImpl::AddAudioCapturer(
467    const scoped_refptr<WebRtcAudioCapturer>& capturer) {
468  DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
469  DCHECK(thread_checker_.CalledOnValidThread());
470  DCHECK(capturer.get());
471  DCHECK(!capturer->device_id().empty());
472  {
473    base::AutoLock auto_lock(lock_);
474    DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
475        capturers_.end());
476    capturers_.push_back(capturer);
477  }
478}
479
480void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
481    const scoped_refptr<WebRtcAudioCapturer>& capturer) {
482  DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
483  DCHECK(thread_checker_.CalledOnValidThread());
484  DCHECK(capturer.get());
485  base::AutoLock auto_lock(lock_);
486  capturers_.remove(capturer);
487}
488
489scoped_refptr<WebRtcAudioCapturer>
490WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
491  base::AutoLock auto_lock(lock_);
492  // Use the last |capturer| which is from the latest getUserMedia call as
493  // the default capture device.
494  return capturers_.empty() ? NULL : capturers_.back();
495}
496
497void WebRtcAudioDeviceImpl::AddPlayoutSink(
498    WebRtcPlayoutDataSource::Sink* sink) {
499  DCHECK(thread_checker_.CalledOnValidThread());
500  DCHECK(sink);
501  base::AutoLock auto_lock(lock_);
502  DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
503      playout_sinks_.end());
504  playout_sinks_.push_back(sink);
505}
506
507void WebRtcAudioDeviceImpl::RemovePlayoutSink(
508    WebRtcPlayoutDataSource::Sink* sink) {
509  DCHECK(thread_checker_.CalledOnValidThread());
510  DCHECK(sink);
511  base::AutoLock auto_lock(lock_);
512  playout_sinks_.remove(sink);
513}
514
515bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
516    int* session_id,
517    int* output_sample_rate,
518    int* output_frames_per_buffer) {
519  DCHECK(thread_checker_.CalledOnValidThread());
520  // If there is no capturer or there are more than one open capture devices,
521  // return false.
522  if (capturers_.empty() || capturers_.size() > 1)
523    return false;
524
525  return GetDefaultCapturer()->GetPairedOutputParameters(
526      session_id, output_sample_rate, output_frames_per_buffer);
527}
528
529}  // namespace content
530