media_stream_audio_processor.cc revision f8ee788a64d60abd8f2d742a5fdedde054ecd910
1f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)// Copyright 2013 The Chromium Authors. All rights reserved. 2f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 3f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)// found in the LICENSE file. 4f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 5f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "content/renderer/media/media_stream_audio_processor.h" 6f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 7f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "base/command_line.h" 8f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "base/debug/trace_event.h" 9effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch#include "base/metrics/histogram.h" 10f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "content/public/common/content_switches.h" 11f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "content/renderer/media/media_stream_audio_processor_options.h" 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "content/renderer/media/rtc_media_constraints.h" 13a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch#include "content/renderer/media/webrtc_audio_device_impl.h" 14f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "media/audio/audio_parameters.h" 15f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "media/base/audio_converter.h" 16f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "media/base/audio_fifo.h" 17f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "media/base/channel_layout.h" 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "third_party/WebKit/public/platform/WebMediaConstraints.h" 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h" 20a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)#include "third_party/webrtc/modules/audio_processing/typing_detection.h" 21f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 22f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)namespace content { 23f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 24f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)namespace { 25f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 26f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)using webrtc::AudioProcessing; 27f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#if defined(OS_ANDROID) 29f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)const int kAudioProcessingSampleRate = 16000; 30f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#else 31f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)const int kAudioProcessingSampleRate = 32000; 32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#endif 33effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochconst int kAudioProcessingNumberOfChannels = 1; 345c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuconst AudioProcessing::ChannelLayout kAudioProcessingChannelLayout = 355c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu AudioProcessing::kMono; 36f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 37f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)const int kMaxNumberOfBuffersInFifo = 2; 38f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 39effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch// Used by UMA histograms and entries shouldn't be re-ordered or removed. 40effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochenum AudioTrackProcessingStates { 41effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch AUDIO_PROCESSING_ENABLED = 0, 42effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch AUDIO_PROCESSING_DISABLED, 43effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch AUDIO_PROCESSING_IN_WEBRTC, 44effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch AUDIO_PROCESSING_MAX 45effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch}; 46effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 47effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochvoid RecordProcessingState(AudioTrackProcessingStates state) { 48effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates", 49effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch state, AUDIO_PROCESSING_MAX); 50effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch} 51effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 52f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} // namespace 53f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 54f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)class MediaStreamAudioProcessor::MediaStreamAudioConverter 55f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) : public media::AudioConverter::InputCallback { 56f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) public: 57f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) MediaStreamAudioConverter(const media::AudioParameters& source_params, 58f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) const media::AudioParameters& sink_params) 59f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) : source_params_(source_params), 60f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) sink_params_(sink_params), 61f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_converter_(source_params, sink_params_, false) { 625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // An instance of MediaStreamAudioConverter may be created in the main 635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // render thread and used in the audio thread, for example, the 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // |MediaStreamAudioProcessor::capture_converter_|. 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) thread_checker_.DetachFromThread(); 66f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_converter_.AddInput(this); 6746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 68f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Create and initialize audio fifo and audio bus wrapper. 69f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // The size of the FIFO should be at least twice of the source buffer size 7046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) // or twice of the sink buffer size. Also, FIFO needs to have enough space 7146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) // to store pre-processed data before passing the data to 7246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) // webrtc::AudioProcessing, which requires 10ms as packet size. 7346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) int max_frame_size = std::max(source_params_.frames_per_buffer(), 7446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) sink_params_.frames_per_buffer()); 75f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) int buffer_size = std::max( 7646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) kMaxNumberOfBuffersInFifo * max_frame_size, 7746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) kMaxNumberOfBuffersInFifo * source_params_.sample_rate() / 100); 78f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size)); 7946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 80f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // TODO(xians): Use CreateWrapper to save one memcpy. 81f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(), 82f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) sink_params_.frames_per_buffer()); 83f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 84f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 85f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) virtual ~MediaStreamAudioConverter() { 86f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_converter_.RemoveInput(this); 87f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 88f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 89f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) void Push(media::AudioBus* audio_source) { 90f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Called on the audio thread, which is the capture audio thread for 91f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread 92f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // for |MediaStreamAudioProcessor::render_converter_|. 93f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // And it must be the same thread as calling Convert(). 94f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 95f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) fifo_->Push(audio_source); 96f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 97f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 98f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) bool Convert(webrtc::AudioFrame* out) { 99f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Called on the audio thread, which is the capture audio thread for 100f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread 101f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // for |MediaStreamAudioProcessor::render_converter_|. 102f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 1035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Return false if there is not enough data in the FIFO, this happens when 1045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // fifo_->frames() / source_params_.sample_rate() is less than 1055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // sink_params.frames_per_buffer() / sink_params.sample_rate(). 1065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (fifo_->frames() * sink_params_.sample_rate() < 1075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sink_params_.frames_per_buffer() * source_params_.sample_rate()) { 108f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return false; 1095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 110f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 1115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Convert data to the output format, this will trigger ProvideInput(). 112f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_converter_.Convert(audio_wrapper_.get()); 113f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 114f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // TODO(xians): Figure out a better way to handle the interleaved and 115f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // deinterleaved format switching. 1165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) DCHECK_EQ(audio_wrapper_->frames(), sink_params_.frames_per_buffer()); 117f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_wrapper_->ToInterleaved(audio_wrapper_->frames(), 118f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) sink_params_.bits_per_sample() / 8, 119f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->data_); 120f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 121f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->samples_per_channel_ = sink_params_.frames_per_buffer(); 122f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->sample_rate_hz_ = sink_params_.sample_rate(); 123f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->speech_type_ = webrtc::AudioFrame::kNormalSpeech; 124f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->vad_activity_ = webrtc::AudioFrame::kVadUnknown; 125f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) out->num_channels_ = sink_params_.channels(); 126f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 127f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return true; 128f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 129f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 130f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) const media::AudioParameters& source_parameters() const { 131f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return source_params_; 132f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 133f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) const media::AudioParameters& sink_parameters() const { 134f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return sink_params_; 135f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 136f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 137f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) private: 138f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // AudioConverter::InputCallback implementation. 139f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) virtual double ProvideInput(media::AudioBus* audio_bus, 140f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::TimeDelta buffer_delay) OVERRIDE { 141f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Called on realtime audio thread. 142f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // TODO(xians): Figure out why the first Convert() triggers ProvideInput 143f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // two times. 144f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (fifo_->frames() < audio_bus->frames()) 145f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return 0; 146f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 147f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) fifo_->Consume(audio_bus, 0, audio_bus->frames()); 148f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 149f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Return 1.0 to indicate no volume scaling on the data. 150f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return 1.0; 151f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 152f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 153f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::ThreadChecker thread_checker_; 154f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) const media::AudioParameters source_params_; 155f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) const media::AudioParameters sink_params_; 156f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 157f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // TODO(xians): consider using SincResampler to save some memcpy. 158f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Handles mixing and resampling between input and output parameters. 159f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::AudioConverter audio_converter_; 160f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) scoped_ptr<media::AudioBus> audio_wrapper_; 161f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) scoped_ptr<media::AudioFifo> fifo_; 162f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)}; 163f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 1640529e5d033099cbfc42635f6f6183833b09dff6eBen Murdochbool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { 165cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return !CommandLine::ForCurrentProcess()->HasSwitch( 166cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) switches::kDisableAudioTrackProcessing); 1670529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch} 1680529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 169f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)MediaStreamAudioProcessor::MediaStreamAudioProcessor( 1705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const blink::WebMediaConstraints& constraints, 171a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) int effects, 172a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) WebRtcPlayoutDataSource* playout_data_source) 1735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) : render_delay_ms_(0), 174a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) playout_data_source_(playout_data_source), 175a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_mirroring_(false), 176a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) typing_detected_(false) { 177f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) capture_thread_checker_.DetachFromThread(); 178f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) render_thread_checker_.DetachFromThread(); 179cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) InitializeAudioProcessingModule(constraints, effects); 180f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if (IsAudioTrackProcessingEnabled()) { 181f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) aec_dump_message_filter_ = AecDumpMessageFilter::Get(); 182f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // In unit tests not creating a message filter, |aec_dump_message_filter_| 183f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // will be NULL. We can just ignore that. Other unit tests and browser tests 184f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) // ensure that we do get the filter when we should. 185f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if (aec_dump_message_filter_) 186f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) aec_dump_message_filter_->AddDelegate(this); 187f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) } 188f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 189f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 190f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { 191f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 192f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if (aec_dump_message_filter_) { 193f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) aec_dump_message_filter_->RemoveDelegate(this); 194f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) aec_dump_message_filter_ = NULL; 195f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) } 196f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) StopAudioProcessing(); 197f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 198f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 199a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void MediaStreamAudioProcessor::OnCaptureFormatChanged( 200a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) const media::AudioParameters& source_params) { 201a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 202a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // There is no need to hold a lock here since the caller guarantees that 203a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks 204a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // on the capture thread. 205a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) InitializeCaptureConverter(source_params); 206f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 207a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // Reset the |capture_thread_checker_| since the capture data will come from 208a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // a new capture thread. 209a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) capture_thread_checker_.DetachFromThread(); 210a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 211f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 212a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void MediaStreamAudioProcessor::PushCaptureData(media::AudioBus* audio_source) { 213a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK(capture_thread_checker_.CalledOnValidThread()); 214a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK_EQ(audio_source->channels(), 215a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) capture_converter_->source_parameters().channels()); 216a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK_EQ(audio_source->frames(), 217a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) capture_converter_->source_parameters().frames_per_buffer()); 218a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 219a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (audio_mirroring_ && 220a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) capture_converter_->source_parameters().channel_layout() == 221a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) media::CHANNEL_LAYOUT_STEREO) { 222a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // Swap the first and second channels. 223a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_source->SwapChannels(0, 1); 224f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 225f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 226a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) capture_converter_->Push(audio_source); 227f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 228f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 229f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)bool MediaStreamAudioProcessor::ProcessAndConsumeData( 230f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::TimeDelta capture_delay, int volume, bool key_pressed, 2315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) int* new_volume, int16** out) { 232f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(capture_thread_checker_.CalledOnValidThread()); 233a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData"); 234f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 235f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (!capture_converter_->Convert(&capture_frame_)) 236f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return false; 237f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 2385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) *new_volume = ProcessData(&capture_frame_, capture_delay, volume, 2395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) key_pressed); 240f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) *out = capture_frame_.data_; 241f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 242f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return true; 243f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 244f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 2455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { 2465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return capture_converter_->source_parameters(); 247f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 248f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 249f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { 250f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return capture_converter_->sink_parameters(); 251f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 252f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 253f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void MediaStreamAudioProcessor::OnAecDumpFile( 254f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) const IPC::PlatformFileForTransit& file_handle) { 255f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 256f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 257f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) base::File file = IPC::PlatformFileForTransitToFile(file_handle); 258f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) DCHECK(file.IsValid()); 259f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 260a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (audio_processing_) 261f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) StartEchoCancellationDump(audio_processing_.get(), file.Pass()); 262f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) else 263f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) file.Close(); 264a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 265a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 266f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void MediaStreamAudioProcessor::OnDisableAecDump() { 267f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 268a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (audio_processing_) 269a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) StopEchoCancellationDump(audio_processing_.get()); 270a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 271a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 272f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)void MediaStreamAudioProcessor::OnIpcClosing() { 273f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 274f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) aec_dump_message_filter_ = NULL; 275f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)} 276f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 277a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus, 278a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) int sample_rate, 279a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) int audio_delay_milliseconds) { 280a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK(render_thread_checker_.CalledOnValidThread()); 281010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) DCHECK(audio_processing_->echo_control_mobile()->is_enabled() ^ 282010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) audio_processing_->echo_cancellation()->is_enabled()); 283a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 284a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData"); 285a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK_LT(audio_delay_milliseconds, 286a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) std::numeric_limits<base::subtle::Atomic32>::max()); 287a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds); 288a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 289a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) InitializeRenderConverterIfNeeded(sample_rate, audio_bus->channels(), 290a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_bus->frames()); 291a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 292a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) render_converter_->Push(audio_bus); 293a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) while (render_converter_->Convert(&render_frame_)) 294a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_processing_->AnalyzeReverseStream(&render_frame_); 295a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 296a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 297a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { 298a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 299a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // There is no need to hold a lock here since the caller guarantees that 300a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // there is no more OnPlayoutData() callback on the render thread. 301a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) render_thread_checker_.DetachFromThread(); 302a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) render_converter_.reset(); 303a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 304a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 305a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { 306a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) stats->typing_noise_detected = 307a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) (base::subtle::Acquire_Load(&typing_detected_) != false); 308a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) GetAecStats(audio_processing_.get(), stats); 309a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)} 310a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 311f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)void MediaStreamAudioProcessor::InitializeAudioProcessingModule( 312cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const blink::WebMediaConstraints& constraints, int effects) { 313f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(!audio_processing_); 314f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 315cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints audio_constraints(constraints, effects); 316a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 317effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch // Audio mirroring can be enabled even though audio processing is otherwise 318effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch // disabled. 319cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) audio_mirroring_ = audio_constraints.GetProperty( 320cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogAudioMirroring); 321effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 322effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch if (!IsAudioTrackProcessingEnabled()) { 323effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); 324effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch return; 325effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch } 326effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 3275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#if defined(OS_IOS) 328cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) // On iOS, VPIO provides built-in AGC and AEC. 329cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool echo_cancellation = false; 330cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_agc = false; 3315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#else 332cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool echo_cancellation = 333cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) audio_constraints.GetEchoCancellationProperty(); 334cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_agc = audio_constraints.GetProperty( 335cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogAutoGainControl); 3365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#endif 3375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#if defined(OS_IOS) || defined(OS_ANDROID) 339cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_experimental_aec = false; 340cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_typing_detection = false; 341f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#else 342cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_experimental_aec = audio_constraints.GetProperty( 343cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogExperimentalEchoCancellation); 344cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_typing_detection = audio_constraints.GetProperty( 345cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogTypingNoiseDetection); 346f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#endif 347f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 348cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_ns = audio_constraints.GetProperty( 349cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogNoiseSuppression); 350cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_experimental_ns = audio_constraints.GetProperty( 351cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogExperimentalNoiseSuppression); 352cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) const bool goog_high_pass_filter = audio_constraints.GetProperty( 353cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) MediaAudioConstraints::kGoogHighpassFilter); 354cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 355cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) // Return immediately if no goog constraint is enabled. 356cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (!echo_cancellation && !goog_experimental_aec && !goog_ns && 357cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) !goog_high_pass_filter && !goog_typing_detection && 358cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) !goog_agc && !goog_experimental_ns) { 359effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch RecordProcessingState(AUDIO_PROCESSING_DISABLED); 360f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return; 361f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 362f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 363f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Create and configure the webrtc::AudioProcessing. 3645c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu audio_processing_.reset(webrtc::AudioProcessing::Create()); 3655c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu CHECK_EQ(0, audio_processing_->Initialize(kAudioProcessingSampleRate, 3665c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu kAudioProcessingSampleRate, 3675c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu kAudioProcessingSampleRate, 3685c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu kAudioProcessingChannelLayout, 3695c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu kAudioProcessingChannelLayout, 3705c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu kAudioProcessingChannelLayout)); 371f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 372f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Enable the audio processing components. 373cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (echo_cancellation) { 374f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) EnableEchoCancellation(audio_processing_.get()); 375cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 376cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_experimental_aec) 377f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) EnableExperimentalEchoCancellation(audio_processing_.get()); 378a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 379a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (playout_data_source_) 380a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) playout_data_source_->AddPlayoutSink(this); 381f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 382f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 383cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_ns) 384f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) EnableNoiseSuppression(audio_processing_.get()); 385f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 386cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_experimental_ns) 387a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) EnableExperimentalNoiseSuppression(audio_processing_.get()); 388a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 389cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_high_pass_filter) 390f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) EnableHighPassFilter(audio_processing_.get()); 391f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 392cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_typing_detection) { 393a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // TODO(xians): Remove this |typing_detector_| after the typing suppression 394a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) // is enabled by default. 395a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) typing_detector_.reset(new webrtc::TypingDetection()); 396a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); 397a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) } 398f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 399cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if (goog_agc) 4005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) EnableAutomaticGainControl(audio_processing_.get()); 401f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 402effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch RecordProcessingState(AUDIO_PROCESSING_ENABLED); 403f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 404f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 4055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)void MediaStreamAudioProcessor::InitializeCaptureConverter( 4065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const media::AudioParameters& source_params) { 407a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) DCHECK(main_thread_checker_.CalledOnValidThread()); 4085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) DCHECK(source_params.IsValid()); 4095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Create and initialize audio converter for the source data. 4115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // When the webrtc AudioProcessing is enabled, the sink format of the 4125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // converter will be the same as the post-processed data format, which is 4135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // 32k mono for desktops and 16k mono for Android. When the AudioProcessing 4145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // is disabled, the sink format will be the same as the source format. 4155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const int sink_sample_rate = audio_processing_ ? 4165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) kAudioProcessingSampleRate : source_params.sample_rate(); 4175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const media::ChannelLayout sink_channel_layout = audio_processing_ ? 418effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch media::GuessChannelLayout(kAudioProcessingNumberOfChannels) : 419a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) source_params.channel_layout(); 4205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // WebRtc AudioProcessing requires 10ms as its packet size. We use this 4225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // native size when processing is enabled. While processing is disabled, and 4235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // the source is running with a buffer size smaller than 10ms buffer, we use 4245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // same buffer size as the incoming format to avoid extra FIFO for WebAudio. 4255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) int sink_buffer_size = sink_sample_rate / 100; 4265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (!audio_processing_ && 4275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) source_params.frames_per_buffer() < sink_buffer_size) { 4285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sink_buffer_size = source_params.frames_per_buffer(); 4295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 4305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) media::AudioParameters sink_params( 4325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout, 4335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sink_sample_rate, 16, sink_buffer_size); 4345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) capture_converter_.reset( 4355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) new MediaStreamAudioConverter(source_params, sink_params)); 4365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)} 4375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 438f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)void MediaStreamAudioProcessor::InitializeRenderConverterIfNeeded( 439f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) int sample_rate, int number_of_channels, int frames_per_buffer) { 440f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(render_thread_checker_.CalledOnValidThread()); 441f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // TODO(xians): Figure out if we need to handle the buffer size change. 442f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (render_converter_.get() && 443f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) render_converter_->source_parameters().sample_rate() == sample_rate && 444f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) render_converter_->source_parameters().channels() == number_of_channels) { 445f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Do nothing if the |render_converter_| has been setup properly. 446f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return; 447f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 448f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 449f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // Create and initialize audio converter for the render data. 450f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // webrtc::AudioProcessing accepts the same format as what it uses to process 451f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) // capture data, which is 32k mono for desktops and 16k mono for Android. 452f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::AudioParameters source_params( 453f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::AudioParameters::AUDIO_PCM_LOW_LATENCY, 454f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::GuessChannelLayout(number_of_channels), sample_rate, 16, 455f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) frames_per_buffer); 456f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::AudioParameters sink_params( 457f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::AudioParameters::AUDIO_PCM_LOW_LATENCY, 458f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, 459f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) kAudioProcessingSampleRate / 100); 460f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) render_converter_.reset( 461f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) new MediaStreamAudioConverter(source_params, sink_params)); 462f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) render_data_bus_ = media::AudioBus::Create(number_of_channels, 463f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) frames_per_buffer); 464f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 465f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 4665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)int MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, 4675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) base::TimeDelta capture_delay, 4685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) int volume, 4695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bool key_pressed) { 470f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK(capture_thread_checker_.CalledOnValidThread()); 471f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (!audio_processing_) 4725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return 0; 473f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 4745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData"); 4755c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu DCHECK_EQ(audio_processing_->input_sample_rate_hz(), 476f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) capture_converter_->sink_parameters().sample_rate()); 477f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK_EQ(audio_processing_->num_input_channels(), 478f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) capture_converter_->sink_parameters().channels()); 479f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK_EQ(audio_processing_->num_output_channels(), 480f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) capture_converter_->sink_parameters().channels()); 481f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 482f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::subtle::Atomic32 render_delay_ms = 483f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::subtle::Acquire_Load(&render_delay_ms_); 484f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) int64 capture_delay_ms = capture_delay.InMilliseconds(); 485f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK_LT(capture_delay_ms, 486f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) std::numeric_limits<base::subtle::Atomic32>::max()); 487f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) int total_delay_ms = capture_delay_ms + render_delay_ms; 4885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (total_delay_ms > 300) { 489f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms 490f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) << "ms; render delay: " << render_delay_ms << "ms"; 491f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) } 492f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 493f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_processing_->set_stream_delay_ms(total_delay_ms); 494a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 495a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel); 496f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) webrtc::GainControl* agc = audio_processing_->gain_control(); 497f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) int err = agc->set_stream_analog_level(volume); 498f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; 499a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 500a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_processing_->set_stream_key_pressed(key_pressed); 501a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 502f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) err = audio_processing_->ProcessStream(audio_frame); 503f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; 504f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 505a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (typing_detector_ && 506a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) audio_frame->vad_activity_ != webrtc::AudioFrame::kVadUnknown) { 507a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool vad_active = 508a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) (audio_frame->vad_activity_ == webrtc::AudioFrame::kVadActive); 509a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) bool typing_detected = typing_detector_->Process(key_pressed, vad_active); 510a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) base::subtle::Release_Store(&typing_detected_, typing_detected); 5115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 5125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 5135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // Return 0 if the volume has not been changed, otherwise return the new 5145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) // volume. 5155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return (agc->stream_analog_level() == volume) ? 5165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 0 : agc->stream_analog_level(); 517f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 518f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 519f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)void MediaStreamAudioProcessor::StopAudioProcessing() { 520f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if (!audio_processing_.get()) 521f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return; 522f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 523f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) StopEchoCancellationDump(audio_processing_.get()); 524a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 525a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if (playout_data_source_) 526a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) playout_data_source_->RemovePlayoutSink(this); 527a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 528f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) audio_processing_.reset(); 529f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} 530f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 531f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)} // namespace content 532