audio_processing_impl.cc revision 949028fbf1e9a01fb96b186b95606c0096e7d13f
1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/audio_processing/audio_processing_impl.h" 12 13#include <assert.h> 14#include <algorithm> 15 16#include "webrtc/base/checks.h" 17#include "webrtc/base/platform_file.h" 18#include "webrtc/common_audio/audio_converter.h" 19#include "webrtc/common_audio/channel_buffer.h" 20#include "webrtc/common_audio/include/audio_util.h" 21#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" 22extern "C" { 23#include "webrtc/modules/audio_processing/aec/aec_core.h" 24} 25#include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" 26#include "webrtc/modules/audio_processing/audio_buffer.h" 27#include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h" 28#include "webrtc/modules/audio_processing/common.h" 29#include "webrtc/modules/audio_processing/echo_cancellation_impl.h" 30#include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" 31#include "webrtc/modules/audio_processing/gain_control_impl.h" 32#include "webrtc/modules/audio_processing/high_pass_filter_impl.h" 33#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h" 34#include "webrtc/modules/audio_processing/level_estimator_impl.h" 35#include "webrtc/modules/audio_processing/noise_suppression_impl.h" 36#include "webrtc/modules/audio_processing/processing_component.h" 37#include "webrtc/modules/audio_processing/transient/transient_suppressor.h" 38#include "webrtc/modules/audio_processing/voice_detection_impl.h" 39#include "webrtc/modules/include/module_common_types.h" 40#include "webrtc/system_wrappers/include/file_wrapper.h" 41#include "webrtc/system_wrappers/include/logging.h" 42#include "webrtc/system_wrappers/include/metrics.h" 43 44#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 45// Files generated at build-time by the protobuf compiler. 46#ifdef WEBRTC_ANDROID_PLATFORM_BUILD 47#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" 48#else 49#include "webrtc/audio_processing/debug.pb.h" 50#endif 51#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 52 53#define RETURN_ON_ERR(expr) \ 54 do { \ 55 int err = (expr); \ 56 if (err != kNoError) { \ 57 return err; \ 58 } \ 59 } while (0) 60 61namespace webrtc { 62namespace { 63 64static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { 65 switch (layout) { 66 case AudioProcessing::kMono: 67 case AudioProcessing::kStereo: 68 return false; 69 case AudioProcessing::kMonoAndKeyboard: 70 case AudioProcessing::kStereoAndKeyboard: 71 return true; 72 } 73 74 assert(false); 75 return false; 76} 77} // namespace 78 79// Throughout webrtc, it's assumed that success is represented by zero. 80static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); 81 82// This class has two main functionalities: 83// 84// 1) It is returned instead of the real GainControl after the new AGC has been 85// enabled in order to prevent an outside user from overriding compression 86// settings. It doesn't do anything in its implementation, except for 87// delegating the const methods and Enable calls to the real GainControl, so 88// AGC can still be disabled. 89// 90// 2) It is injected into AgcManagerDirect and implements volume callbacks for 91// getting and setting the volume level. It just caches this value to be used 92// in VoiceEngine later. 93class GainControlForNewAgc : public GainControl, public VolumeCallbacks { 94 public: 95 explicit GainControlForNewAgc(GainControlImpl* gain_control) 96 : real_gain_control_(gain_control), volume_(0) {} 97 98 // GainControl implementation. 99 int Enable(bool enable) override { 100 return real_gain_control_->Enable(enable); 101 } 102 bool is_enabled() const override { return real_gain_control_->is_enabled(); } 103 int set_stream_analog_level(int level) override { 104 volume_ = level; 105 return AudioProcessing::kNoError; 106 } 107 int stream_analog_level() override { return volume_; } 108 int set_mode(Mode mode) override { return AudioProcessing::kNoError; } 109 Mode mode() const override { return GainControl::kAdaptiveAnalog; } 110 int set_target_level_dbfs(int level) override { 111 return AudioProcessing::kNoError; 112 } 113 int target_level_dbfs() const override { 114 return real_gain_control_->target_level_dbfs(); 115 } 116 int set_compression_gain_db(int gain) override { 117 return AudioProcessing::kNoError; 118 } 119 int compression_gain_db() const override { 120 return real_gain_control_->compression_gain_db(); 121 } 122 int enable_limiter(bool enable) override { return AudioProcessing::kNoError; } 123 bool is_limiter_enabled() const override { 124 return real_gain_control_->is_limiter_enabled(); 125 } 126 int set_analog_level_limits(int minimum, int maximum) override { 127 return AudioProcessing::kNoError; 128 } 129 int analog_level_minimum() const override { 130 return real_gain_control_->analog_level_minimum(); 131 } 132 int analog_level_maximum() const override { 133 return real_gain_control_->analog_level_maximum(); 134 } 135 bool stream_is_saturated() const override { 136 return real_gain_control_->stream_is_saturated(); 137 } 138 139 // VolumeCallbacks implementation. 140 void SetMicVolume(int volume) override { volume_ = volume; } 141 int GetMicVolume() override { return volume_; } 142 143 private: 144 GainControl* real_gain_control_; 145 int volume_; 146}; 147 148struct AudioProcessingImpl::ApmPublicSubmodules { 149 ApmPublicSubmodules() 150 : echo_cancellation(nullptr), 151 echo_control_mobile(nullptr), 152 gain_control(nullptr), 153 voice_detection(nullptr) {} 154 // Accessed externally of APM without any lock acquired. 155 EchoCancellationImpl* echo_cancellation; 156 EchoControlMobileImpl* echo_control_mobile; 157 GainControlImpl* gain_control; 158 rtc::scoped_ptr<HighPassFilterImpl> high_pass_filter; 159 rtc::scoped_ptr<LevelEstimatorImpl> level_estimator; 160 rtc::scoped_ptr<NoiseSuppressionImpl> noise_suppression; 161 VoiceDetectionImpl* voice_detection; 162 rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc; 163 164 // Accessed internally from both render and capture. 165 rtc::scoped_ptr<TransientSuppressor> transient_suppressor; 166 rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer; 167}; 168 169struct AudioProcessingImpl::ApmPrivateSubmodules { 170 explicit ApmPrivateSubmodules(Beamformer<float>* beamformer) 171 : beamformer(beamformer) {} 172 // Accessed internally from capture or during initialization 173 std::list<ProcessingComponent*> component_list; 174 rtc::scoped_ptr<Beamformer<float>> beamformer; 175 rtc::scoped_ptr<AgcManagerDirect> agc_manager; 176}; 177 178const int AudioProcessing::kNativeSampleRatesHz[] = { 179 AudioProcessing::kSampleRate8kHz, 180 AudioProcessing::kSampleRate16kHz, 181 AudioProcessing::kSampleRate32kHz, 182 AudioProcessing::kSampleRate48kHz}; 183const size_t AudioProcessing::kNumNativeSampleRates = 184 arraysize(AudioProcessing::kNativeSampleRatesHz); 185const int AudioProcessing::kMaxNativeSampleRateHz = AudioProcessing:: 186 kNativeSampleRatesHz[AudioProcessing::kNumNativeSampleRates - 1]; 187const int AudioProcessing::kMaxAECMSampleRateHz = kSampleRate16kHz; 188 189AudioProcessing* AudioProcessing::Create() { 190 Config config; 191 return Create(config, nullptr); 192} 193 194AudioProcessing* AudioProcessing::Create(const Config& config) { 195 return Create(config, nullptr); 196} 197 198AudioProcessing* AudioProcessing::Create(const Config& config, 199 Beamformer<float>* beamformer) { 200 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); 201 if (apm->Initialize() != kNoError) { 202 delete apm; 203 apm = nullptr; 204 } 205 206 return apm; 207} 208 209AudioProcessingImpl::AudioProcessingImpl(const Config& config) 210 : AudioProcessingImpl(config, nullptr) {} 211 212AudioProcessingImpl::AudioProcessingImpl(const Config& config, 213 Beamformer<float>* beamformer) 214 : public_submodules_(new ApmPublicSubmodules()), 215 private_submodules_(new ApmPrivateSubmodules(beamformer)), 216 constants_(config.Get<ExperimentalAgc>().startup_min_volume, 217 config.Get<Beamforming>().array_geometry, 218 config.Get<Beamforming>().target_direction, 219#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) 220 false, 221#else 222 config.Get<ExperimentalAgc>().enabled, 223#endif 224 config.Get<Intelligibility>().enabled, 225 config.Get<Beamforming>().enabled), 226 227#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) 228 capture_(false) 229#else 230 capture_(config.Get<ExperimentalNs>().enabled) 231#endif 232{ 233 { 234 rtc::CritScope cs_render(&crit_render_); 235 rtc::CritScope cs_capture(&crit_capture_); 236 237 public_submodules_->echo_cancellation = 238 new EchoCancellationImpl(this, &crit_render_, &crit_capture_); 239 public_submodules_->echo_control_mobile = 240 new EchoControlMobileImpl(this, &crit_render_, &crit_capture_); 241 public_submodules_->gain_control = 242 new GainControlImpl(this, &crit_capture_, &crit_capture_); 243 public_submodules_->high_pass_filter.reset( 244 new HighPassFilterImpl(&crit_capture_)); 245 public_submodules_->level_estimator.reset( 246 new LevelEstimatorImpl(&crit_capture_)); 247 public_submodules_->noise_suppression.reset( 248 new NoiseSuppressionImpl(&crit_capture_)); 249 public_submodules_->voice_detection = 250 new VoiceDetectionImpl(this, &crit_capture_); 251 public_submodules_->gain_control_for_new_agc.reset( 252 new GainControlForNewAgc(public_submodules_->gain_control)); 253 254 private_submodules_->component_list.push_back( 255 public_submodules_->echo_cancellation); 256 private_submodules_->component_list.push_back( 257 public_submodules_->echo_control_mobile); 258 private_submodules_->component_list.push_back( 259 public_submodules_->gain_control); 260 private_submodules_->component_list.push_back( 261 public_submodules_->voice_detection); 262 } 263 264 SetExtraOptions(config); 265} 266 267AudioProcessingImpl::~AudioProcessingImpl() { 268 // Depends on gain_control_ and 269 // public_submodules_->gain_control_for_new_agc. 270 private_submodules_->agc_manager.reset(); 271 // Depends on gain_control_. 272 public_submodules_->gain_control_for_new_agc.reset(); 273 while (!private_submodules_->component_list.empty()) { 274 ProcessingComponent* component = 275 private_submodules_->component_list.front(); 276 component->Destroy(); 277 delete component; 278 private_submodules_->component_list.pop_front(); 279 } 280 281#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 282 if (debug_dump_.debug_file->Open()) { 283 debug_dump_.debug_file->CloseFile(); 284 } 285#endif 286} 287 288int AudioProcessingImpl::Initialize() { 289 // Run in a single-threaded manner during initialization. 290 rtc::CritScope cs_render(&crit_render_); 291 rtc::CritScope cs_capture(&crit_capture_); 292 return InitializeLocked(); 293} 294 295int AudioProcessingImpl::Initialize(int input_sample_rate_hz, 296 int output_sample_rate_hz, 297 int reverse_sample_rate_hz, 298 ChannelLayout input_layout, 299 ChannelLayout output_layout, 300 ChannelLayout reverse_layout) { 301 const ProcessingConfig processing_config = { 302 {{input_sample_rate_hz, 303 ChannelsFromLayout(input_layout), 304 LayoutHasKeyboard(input_layout)}, 305 {output_sample_rate_hz, 306 ChannelsFromLayout(output_layout), 307 LayoutHasKeyboard(output_layout)}, 308 {reverse_sample_rate_hz, 309 ChannelsFromLayout(reverse_layout), 310 LayoutHasKeyboard(reverse_layout)}, 311 {reverse_sample_rate_hz, 312 ChannelsFromLayout(reverse_layout), 313 LayoutHasKeyboard(reverse_layout)}}}; 314 315 return Initialize(processing_config); 316} 317 318int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { 319 // Run in a single-threaded manner during initialization. 320 rtc::CritScope cs_render(&crit_render_); 321 rtc::CritScope cs_capture(&crit_capture_); 322 return InitializeLocked(processing_config); 323} 324 325int AudioProcessingImpl::MaybeInitializeRender( 326 const ProcessingConfig& processing_config) { 327 return MaybeInitialize(processing_config); 328} 329 330int AudioProcessingImpl::MaybeInitializeCapture( 331 const ProcessingConfig& processing_config) { 332 return MaybeInitialize(processing_config); 333} 334 335// Calls InitializeLocked() if any of the audio parameters have changed from 336// their current values (needs to be called while holding the crit_render_lock). 337int AudioProcessingImpl::MaybeInitialize( 338 const ProcessingConfig& processing_config) { 339 // Called from both threads. Thread check is therefore not possible. 340 if (processing_config == formats_.api_format) { 341 return kNoError; 342 } 343 344 rtc::CritScope cs_capture(&crit_capture_); 345 return InitializeLocked(processing_config); 346} 347 348int AudioProcessingImpl::InitializeLocked() { 349 const int fwd_audio_buffer_channels = 350 constants_.beamformer_enabled 351 ? formats_.api_format.input_stream().num_channels() 352 : formats_.api_format.output_stream().num_channels(); 353 const int rev_audio_buffer_out_num_frames = 354 formats_.api_format.reverse_output_stream().num_frames() == 0 355 ? formats_.rev_proc_format.num_frames() 356 : formats_.api_format.reverse_output_stream().num_frames(); 357 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { 358 render_.render_audio.reset(new AudioBuffer( 359 formats_.api_format.reverse_input_stream().num_frames(), 360 formats_.api_format.reverse_input_stream().num_channels(), 361 formats_.rev_proc_format.num_frames(), 362 formats_.rev_proc_format.num_channels(), 363 rev_audio_buffer_out_num_frames)); 364 if (rev_conversion_needed()) { 365 render_.render_converter = AudioConverter::Create( 366 formats_.api_format.reverse_input_stream().num_channels(), 367 formats_.api_format.reverse_input_stream().num_frames(), 368 formats_.api_format.reverse_output_stream().num_channels(), 369 formats_.api_format.reverse_output_stream().num_frames()); 370 } else { 371 render_.render_converter.reset(nullptr); 372 } 373 } else { 374 render_.render_audio.reset(nullptr); 375 render_.render_converter.reset(nullptr); 376 } 377 capture_.capture_audio.reset( 378 new AudioBuffer(formats_.api_format.input_stream().num_frames(), 379 formats_.api_format.input_stream().num_channels(), 380 capture_nonlocked_.fwd_proc_format.num_frames(), 381 fwd_audio_buffer_channels, 382 formats_.api_format.output_stream().num_frames())); 383 384 // Initialize all components. 385 for (auto item : private_submodules_->component_list) { 386 int err = item->Initialize(); 387 if (err != kNoError) { 388 return err; 389 } 390 } 391 392 InitializeExperimentalAgc(); 393 InitializeTransient(); 394 InitializeBeamformer(); 395 InitializeIntelligibility(); 396 InitializeHighPassFilter(); 397 InitializeNoiseSuppression(); 398 InitializeLevelEstimator(); 399 400#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 401 if (debug_dump_.debug_file->Open()) { 402 int err = WriteInitMessage(); 403 if (err != kNoError) { 404 return err; 405 } 406 } 407#endif 408 409 return kNoError; 410} 411 412int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { 413 for (const auto& stream : config.streams) { 414 if (stream.num_channels() < 0) { 415 return kBadNumberChannelsError; 416 } 417 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { 418 return kBadSampleRateError; 419 } 420 } 421 422 const int num_in_channels = config.input_stream().num_channels(); 423 const int num_out_channels = config.output_stream().num_channels(); 424 425 // Need at least one input channel. 426 // Need either one output channel or as many outputs as there are inputs. 427 if (num_in_channels == 0 || 428 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { 429 return kBadNumberChannelsError; 430 } 431 432 if (constants_.beamformer_enabled && (static_cast<size_t>(num_in_channels) != 433 constants_.array_geometry.size() || 434 num_out_channels > 1)) { 435 return kBadNumberChannelsError; 436 } 437 438 formats_.api_format = config; 439 440 // We process at the closest native rate >= min(input rate, output rate)... 441 const int min_proc_rate = 442 std::min(formats_.api_format.input_stream().sample_rate_hz(), 443 formats_.api_format.output_stream().sample_rate_hz()); 444 int fwd_proc_rate; 445 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { 446 fwd_proc_rate = kNativeSampleRatesHz[i]; 447 if (fwd_proc_rate >= min_proc_rate) { 448 break; 449 } 450 } 451 // ...with one exception. 452 if (public_submodules_->echo_control_mobile->is_enabled() && 453 min_proc_rate > kMaxAECMSampleRateHz) { 454 fwd_proc_rate = kMaxAECMSampleRateHz; 455 } 456 457 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); 458 459 // We normally process the reverse stream at 16 kHz. Unless... 460 int rev_proc_rate = kSampleRate16kHz; 461 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { 462 // ...the forward stream is at 8 kHz. 463 rev_proc_rate = kSampleRate8kHz; 464 } else { 465 if (formats_.api_format.reverse_input_stream().sample_rate_hz() == 466 kSampleRate32kHz) { 467 // ...or the input is at 32 kHz, in which case we use the splitting 468 // filter rather than the resampler. 469 rev_proc_rate = kSampleRate32kHz; 470 } 471 } 472 473 // Always downmix the reverse stream to mono for analysis. This has been 474 // demonstrated to work well for AEC in most practical scenarios. 475 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); 476 477 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || 478 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { 479 capture_nonlocked_.split_rate = kSampleRate16kHz; 480 } else { 481 capture_nonlocked_.split_rate = 482 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); 483 } 484 485 return InitializeLocked(); 486} 487 488void AudioProcessingImpl::SetExtraOptions(const Config& config) { 489 // Run in a single-threaded manner when setting the extra options. 490 rtc::CritScope cs_render(&crit_render_); 491 rtc::CritScope cs_capture(&crit_capture_); 492 for (auto item : private_submodules_->component_list) { 493 item->SetExtraOptions(config); 494 } 495 496 if (capture_.transient_suppressor_enabled != 497 config.Get<ExperimentalNs>().enabled) { 498 capture_.transient_suppressor_enabled = 499 config.Get<ExperimentalNs>().enabled; 500 InitializeTransient(); 501 } 502} 503 504int AudioProcessingImpl::proc_sample_rate_hz() const { 505 // Used as callback from submodules, hence locking is not allowed. 506 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); 507} 508 509int AudioProcessingImpl::proc_split_sample_rate_hz() const { 510 // Used as callback from submodules, hence locking is not allowed. 511 return capture_nonlocked_.split_rate; 512} 513 514int AudioProcessingImpl::num_reverse_channels() const { 515 // Used as callback from submodules, hence locking is not allowed. 516 return formats_.rev_proc_format.num_channels(); 517} 518 519int AudioProcessingImpl::num_input_channels() const { 520 // Used as callback from submodules, hence locking is not allowed. 521 return formats_.api_format.input_stream().num_channels(); 522} 523 524int AudioProcessingImpl::num_output_channels() const { 525 // Used as callback from submodules, hence locking is not allowed. 526 return formats_.api_format.output_stream().num_channels(); 527} 528 529void AudioProcessingImpl::set_output_will_be_muted(bool muted) { 530 rtc::CritScope cs(&crit_capture_); 531 capture_.output_will_be_muted = muted; 532 if (private_submodules_->agc_manager.get()) { 533 private_submodules_->agc_manager->SetCaptureMuted( 534 capture_.output_will_be_muted); 535 } 536} 537 538 539int AudioProcessingImpl::ProcessStream(const float* const* src, 540 size_t samples_per_channel, 541 int input_sample_rate_hz, 542 ChannelLayout input_layout, 543 int output_sample_rate_hz, 544 ChannelLayout output_layout, 545 float* const* dest) { 546 StreamConfig input_stream; 547 StreamConfig output_stream; 548 { 549 // Access the formats_.api_format.input_stream beneath the capture lock. 550 // The lock must be released as it is later required in the call 551 // to ProcessStream(,,,); 552 rtc::CritScope cs(&crit_capture_); 553 input_stream = formats_.api_format.input_stream(); 554 output_stream = formats_.api_format.output_stream(); 555 } 556 557 input_stream.set_sample_rate_hz(input_sample_rate_hz); 558 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); 559 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); 560 output_stream.set_sample_rate_hz(output_sample_rate_hz); 561 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); 562 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); 563 564 if (samples_per_channel != input_stream.num_frames()) { 565 return kBadDataLengthError; 566 } 567 return ProcessStream(src, input_stream, output_stream, dest); 568} 569 570int AudioProcessingImpl::ProcessStream(const float* const* src, 571 const StreamConfig& input_config, 572 const StreamConfig& output_config, 573 float* const* dest) { 574 ProcessingConfig processing_config; 575 { 576 // Acquire the capture lock in order to safely call the function 577 // that retrieves the render side data. This function accesses apm 578 // getters that need the capture lock held when being called. 579 rtc::CritScope cs_capture(&crit_capture_); 580 public_submodules_->echo_cancellation->ReadQueuedRenderData(); 581 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); 582 public_submodules_->gain_control->ReadQueuedRenderData(); 583 584 if (!src || !dest) { 585 return kNullPointerError; 586 } 587 588 processing_config = formats_.api_format; 589 } 590 591 processing_config.input_stream() = input_config; 592 processing_config.output_stream() = output_config; 593 594 { 595 // Do conditional reinitialization. 596 rtc::CritScope cs_render(&crit_render_); 597 RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); 598 } 599 rtc::CritScope cs_capture(&crit_capture_); 600 assert(processing_config.input_stream().num_frames() == 601 formats_.api_format.input_stream().num_frames()); 602 603#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 604 if (debug_dump_.debug_file->Open()) { 605 RETURN_ON_ERR(WriteConfigMessage(false)); 606 607 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); 608 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); 609 const size_t channel_size = 610 sizeof(float) * formats_.api_format.input_stream().num_frames(); 611 for (int i = 0; i < formats_.api_format.input_stream().num_channels(); ++i) 612 msg->add_input_channel(src[i], channel_size); 613 } 614#endif 615 616 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); 617 RETURN_ON_ERR(ProcessStreamLocked()); 618 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); 619 620#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 621 if (debug_dump_.debug_file->Open()) { 622 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); 623 const size_t channel_size = 624 sizeof(float) * formats_.api_format.output_stream().num_frames(); 625 for (int i = 0; i < formats_.api_format.output_stream().num_channels(); ++i) 626 msg->add_output_channel(dest[i], channel_size); 627 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 628 &crit_debug_, &debug_dump_.capture)); 629 } 630#endif 631 632 return kNoError; 633} 634 635int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { 636 { 637 // Acquire the capture lock in order to safely call the function 638 // that retrieves the render side data. This function accesses apm 639 // getters that need the capture lock held when being called. 640 // The lock needs to be released as 641 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock 642 // as well. 643 rtc::CritScope cs_capture(&crit_capture_); 644 public_submodules_->echo_cancellation->ReadQueuedRenderData(); 645 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); 646 public_submodules_->gain_control->ReadQueuedRenderData(); 647 } 648 649 if (!frame) { 650 return kNullPointerError; 651 } 652 // Must be a native rate. 653 if (frame->sample_rate_hz_ != kSampleRate8kHz && 654 frame->sample_rate_hz_ != kSampleRate16kHz && 655 frame->sample_rate_hz_ != kSampleRate32kHz && 656 frame->sample_rate_hz_ != kSampleRate48kHz) { 657 return kBadSampleRateError; 658 } 659 660 if (public_submodules_->echo_control_mobile->is_enabled() && 661 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { 662 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; 663 return kUnsupportedComponentError; 664 } 665 666 ProcessingConfig processing_config; 667 { 668 // Aquire lock for the access of api_format. 669 // The lock is released immediately due to the conditional 670 // reinitialization. 671 rtc::CritScope cs_capture(&crit_capture_); 672 // TODO(ajm): The input and output rates and channels are currently 673 // constrained to be identical in the int16 interface. 674 processing_config = formats_.api_format; 675 } 676 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); 677 processing_config.input_stream().set_num_channels(frame->num_channels_); 678 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); 679 processing_config.output_stream().set_num_channels(frame->num_channels_); 680 681 { 682 // Do conditional reinitialization. 683 rtc::CritScope cs_render(&crit_render_); 684 RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); 685 } 686 rtc::CritScope cs_capture(&crit_capture_); 687 if (frame->samples_per_channel_ != 688 formats_.api_format.input_stream().num_frames()) { 689 return kBadDataLengthError; 690 } 691 692#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 693 if (debug_dump_.debug_file->Open()) { 694 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); 695 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); 696 const size_t data_size = 697 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 698 msg->set_input_data(frame->data_, data_size); 699 } 700#endif 701 702 capture_.capture_audio->DeinterleaveFrom(frame); 703 RETURN_ON_ERR(ProcessStreamLocked()); 704 capture_.capture_audio->InterleaveTo(frame, 705 output_copy_needed(is_data_processed())); 706 707#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 708 if (debug_dump_.debug_file->Open()) { 709 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); 710 const size_t data_size = 711 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 712 msg->set_output_data(frame->data_, data_size); 713 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 714 &crit_debug_, &debug_dump_.capture)); 715 } 716#endif 717 718 return kNoError; 719} 720 721int AudioProcessingImpl::ProcessStreamLocked() { 722#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 723 if (debug_dump_.debug_file->Open()) { 724 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); 725 msg->set_delay(capture_nonlocked_.stream_delay_ms); 726 msg->set_drift( 727 public_submodules_->echo_cancellation->stream_drift_samples()); 728 msg->set_level(gain_control()->stream_analog_level()); 729 msg->set_keypress(capture_.key_pressed); 730 } 731#endif 732 733 MaybeUpdateHistograms(); 734 735 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. 736 737 if (constants_.use_new_agc && 738 public_submodules_->gain_control->is_enabled()) { 739 private_submodules_->agc_manager->AnalyzePreProcess( 740 ca->channels()[0], ca->num_channels(), 741 capture_nonlocked_.fwd_proc_format.num_frames()); 742 } 743 744 bool data_processed = is_data_processed(); 745 if (analysis_needed(data_processed)) { 746 ca->SplitIntoFrequencyBands(); 747 } 748 749 if (constants_.intelligibility_enabled) { 750 public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio( 751 ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, 752 ca->num_channels()); 753 } 754 755 if (constants_.beamformer_enabled) { 756 private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(), 757 ca->split_data_f()); 758 ca->set_num_channels(1); 759 } 760 761 public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); 762 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); 763 public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); 764 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); 765 766 if (public_submodules_->echo_control_mobile->is_enabled() && 767 public_submodules_->noise_suppression->is_enabled()) { 768 ca->CopyLowPassToReference(); 769 } 770 public_submodules_->noise_suppression->ProcessCaptureAudio(ca); 771 RETURN_ON_ERR( 772 public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); 773 RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca)); 774 775 if (constants_.use_new_agc && 776 public_submodules_->gain_control->is_enabled() && 777 (!constants_.beamformer_enabled || 778 private_submodules_->beamformer->is_target_present())) { 779 private_submodules_->agc_manager->Process( 780 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), 781 capture_nonlocked_.split_rate); 782 } 783 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca)); 784 785 if (synthesis_needed(data_processed)) { 786 ca->MergeFrequencyBands(); 787 } 788 789 // TODO(aluebs): Investigate if the transient suppression placement should be 790 // before or after the AGC. 791 if (capture_.transient_suppressor_enabled) { 792 float voice_probability = 793 private_submodules_->agc_manager.get() 794 ? private_submodules_->agc_manager->voice_probability() 795 : 1.f; 796 797 public_submodules_->transient_suppressor->Suppress( 798 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), 799 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), 800 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, 801 capture_.key_pressed); 802 } 803 804 // The level estimator operates on the recombined data. 805 public_submodules_->level_estimator->ProcessStream(ca); 806 807 capture_.was_stream_delay_set = false; 808 return kNoError; 809} 810 811int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, 812 size_t samples_per_channel, 813 int rev_sample_rate_hz, 814 ChannelLayout layout) { 815 rtc::CritScope cs(&crit_render_); 816 const StreamConfig reverse_config = { 817 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), 818 }; 819 if (samples_per_channel != reverse_config.num_frames()) { 820 return kBadDataLengthError; 821 } 822 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); 823} 824 825int AudioProcessingImpl::ProcessReverseStream( 826 const float* const* src, 827 const StreamConfig& reverse_input_config, 828 const StreamConfig& reverse_output_config, 829 float* const* dest) { 830 rtc::CritScope cs(&crit_render_); 831 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, 832 reverse_output_config)); 833 if (is_rev_processed()) { 834 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), 835 dest); 836 } else if (render_check_rev_conversion_needed()) { 837 render_.render_converter->Convert(src, reverse_input_config.num_samples(), 838 dest, 839 reverse_output_config.num_samples()); 840 } else { 841 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), 842 reverse_input_config.num_channels(), dest); 843 } 844 845 return kNoError; 846} 847 848int AudioProcessingImpl::AnalyzeReverseStreamLocked( 849 const float* const* src, 850 const StreamConfig& reverse_input_config, 851 const StreamConfig& reverse_output_config) { 852 if (src == nullptr) { 853 return kNullPointerError; 854 } 855 856 if (reverse_input_config.num_channels() <= 0) { 857 return kBadNumberChannelsError; 858 } 859 860 ProcessingConfig processing_config = formats_.api_format; 861 processing_config.reverse_input_stream() = reverse_input_config; 862 processing_config.reverse_output_stream() = reverse_output_config; 863 864 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); 865 assert(reverse_input_config.num_frames() == 866 formats_.api_format.reverse_input_stream().num_frames()); 867 868#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 869 if (debug_dump_.debug_file->Open()) { 870 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); 871 audioproc::ReverseStream* msg = 872 debug_dump_.render.event_msg->mutable_reverse_stream(); 873 const size_t channel_size = 874 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); 875 for (int i = 0; 876 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) 877 msg->add_channel(src[i], channel_size); 878 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 879 &crit_debug_, &debug_dump_.render)); 880 } 881#endif 882 883 render_.render_audio->CopyFrom(src, 884 formats_.api_format.reverse_input_stream()); 885 return ProcessReverseStreamLocked(); 886} 887 888int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { 889 RETURN_ON_ERR(AnalyzeReverseStream(frame)); 890 rtc::CritScope cs(&crit_render_); 891 if (is_rev_processed()) { 892 render_.render_audio->InterleaveTo(frame, true); 893 } 894 895 return kNoError; 896} 897 898int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { 899 rtc::CritScope cs(&crit_render_); 900 if (frame == nullptr) { 901 return kNullPointerError; 902 } 903 // Must be a native rate. 904 if (frame->sample_rate_hz_ != kSampleRate8kHz && 905 frame->sample_rate_hz_ != kSampleRate16kHz && 906 frame->sample_rate_hz_ != kSampleRate32kHz && 907 frame->sample_rate_hz_ != kSampleRate48kHz) { 908 return kBadSampleRateError; 909 } 910 // This interface does not tolerate different forward and reverse rates. 911 if (frame->sample_rate_hz_ != 912 formats_.api_format.input_stream().sample_rate_hz()) { 913 return kBadSampleRateError; 914 } 915 916 if (frame->num_channels_ <= 0) { 917 return kBadNumberChannelsError; 918 } 919 920 ProcessingConfig processing_config = formats_.api_format; 921 processing_config.reverse_input_stream().set_sample_rate_hz( 922 frame->sample_rate_hz_); 923 processing_config.reverse_input_stream().set_num_channels( 924 frame->num_channels_); 925 processing_config.reverse_output_stream().set_sample_rate_hz( 926 frame->sample_rate_hz_); 927 processing_config.reverse_output_stream().set_num_channels( 928 frame->num_channels_); 929 930 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); 931 if (frame->samples_per_channel_ != 932 formats_.api_format.reverse_input_stream().num_frames()) { 933 return kBadDataLengthError; 934 } 935 936#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 937 if (debug_dump_.debug_file->Open()) { 938 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); 939 audioproc::ReverseStream* msg = 940 debug_dump_.render.event_msg->mutable_reverse_stream(); 941 const size_t data_size = 942 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 943 msg->set_data(frame->data_, data_size); 944 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 945 &crit_debug_, &debug_dump_.render)); 946 } 947#endif 948 render_.render_audio->DeinterleaveFrom(frame); 949 return ProcessReverseStreamLocked(); 950} 951 952int AudioProcessingImpl::ProcessReverseStreamLocked() { 953 AudioBuffer* ra = render_.render_audio.get(); // For brevity. 954 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) { 955 ra->SplitIntoFrequencyBands(); 956 } 957 958 if (constants_.intelligibility_enabled) { 959 // Currently run in single-threaded mode when the intelligibility 960 // enhancer is activated. 961 // TODO(peah): Fix to be properly multi-threaded. 962 rtc::CritScope cs(&crit_capture_); 963 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( 964 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, 965 ra->num_channels()); 966 } 967 968 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); 969 RETURN_ON_ERR( 970 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); 971 if (!constants_.use_new_agc) { 972 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); 973 } 974 975 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz && 976 is_rev_processed()) { 977 ra->MergeFrequencyBands(); 978 } 979 980 return kNoError; 981} 982 983int AudioProcessingImpl::set_stream_delay_ms(int delay) { 984 rtc::CritScope cs(&crit_capture_); 985 Error retval = kNoError; 986 capture_.was_stream_delay_set = true; 987 delay += capture_.delay_offset_ms; 988 989 if (delay < 0) { 990 delay = 0; 991 retval = kBadStreamParameterWarning; 992 } 993 994 // TODO(ajm): the max is rather arbitrarily chosen; investigate. 995 if (delay > 500) { 996 delay = 500; 997 retval = kBadStreamParameterWarning; 998 } 999 1000 capture_nonlocked_.stream_delay_ms = delay; 1001 return retval; 1002} 1003 1004int AudioProcessingImpl::stream_delay_ms() const { 1005 // Used as callback from submodules, hence locking is not allowed. 1006 return capture_nonlocked_.stream_delay_ms; 1007} 1008 1009bool AudioProcessingImpl::was_stream_delay_set() const { 1010 // Used as callback from submodules, hence locking is not allowed. 1011 return capture_.was_stream_delay_set; 1012} 1013 1014void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { 1015 rtc::CritScope cs(&crit_capture_); 1016 capture_.key_pressed = key_pressed; 1017} 1018 1019void AudioProcessingImpl::set_delay_offset_ms(int offset) { 1020 rtc::CritScope cs(&crit_capture_); 1021 capture_.delay_offset_ms = offset; 1022} 1023 1024int AudioProcessingImpl::delay_offset_ms() const { 1025 rtc::CritScope cs(&crit_capture_); 1026 return capture_.delay_offset_ms; 1027} 1028 1029int AudioProcessingImpl::StartDebugRecording( 1030 const char filename[AudioProcessing::kMaxFilenameSize]) { 1031 // Run in a single-threaded manner. 1032 rtc::CritScope cs_render(&crit_render_); 1033 rtc::CritScope cs_capture(&crit_capture_); 1034 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); 1035 1036 if (filename == nullptr) { 1037 return kNullPointerError; 1038 } 1039 1040#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1041 // Stop any ongoing recording. 1042 if (debug_dump_.debug_file->Open()) { 1043 if (debug_dump_.debug_file->CloseFile() == -1) { 1044 return kFileError; 1045 } 1046 } 1047 1048 if (debug_dump_.debug_file->OpenFile(filename, false) == -1) { 1049 debug_dump_.debug_file->CloseFile(); 1050 return kFileError; 1051 } 1052 1053 RETURN_ON_ERR(WriteConfigMessage(true)); 1054 RETURN_ON_ERR(WriteInitMessage()); 1055 return kNoError; 1056#else 1057 return kUnsupportedFunctionError; 1058#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1059} 1060 1061int AudioProcessingImpl::StartDebugRecording(FILE* handle) { 1062 // Run in a single-threaded manner. 1063 rtc::CritScope cs_render(&crit_render_); 1064 rtc::CritScope cs_capture(&crit_capture_); 1065 1066 if (handle == nullptr) { 1067 return kNullPointerError; 1068 } 1069 1070#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1071 // Stop any ongoing recording. 1072 if (debug_dump_.debug_file->Open()) { 1073 if (debug_dump_.debug_file->CloseFile() == -1) { 1074 return kFileError; 1075 } 1076 } 1077 1078 if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) { 1079 return kFileError; 1080 } 1081 1082 RETURN_ON_ERR(WriteConfigMessage(true)); 1083 RETURN_ON_ERR(WriteInitMessage()); 1084 return kNoError; 1085#else 1086 return kUnsupportedFunctionError; 1087#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1088} 1089 1090int AudioProcessingImpl::StartDebugRecordingForPlatformFile( 1091 rtc::PlatformFile handle) { 1092 // Run in a single-threaded manner. 1093 rtc::CritScope cs_render(&crit_render_); 1094 rtc::CritScope cs_capture(&crit_capture_); 1095 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); 1096 return StartDebugRecording(stream); 1097} 1098 1099int AudioProcessingImpl::StopDebugRecording() { 1100 // Run in a single-threaded manner. 1101 rtc::CritScope cs_render(&crit_render_); 1102 rtc::CritScope cs_capture(&crit_capture_); 1103 1104#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1105 // We just return if recording hasn't started. 1106 if (debug_dump_.debug_file->Open()) { 1107 if (debug_dump_.debug_file->CloseFile() == -1) { 1108 return kFileError; 1109 } 1110 } 1111 return kNoError; 1112#else 1113 return kUnsupportedFunctionError; 1114#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1115} 1116 1117EchoCancellation* AudioProcessingImpl::echo_cancellation() const { 1118 // Adding a lock here has no effect as it allows any access to the submodule 1119 // from the returned pointer. 1120 return public_submodules_->echo_cancellation; 1121} 1122 1123EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { 1124 // Adding a lock here has no effect as it allows any access to the submodule 1125 // from the returned pointer. 1126 return public_submodules_->echo_control_mobile; 1127} 1128 1129GainControl* AudioProcessingImpl::gain_control() const { 1130 // Adding a lock here has no effect as it allows any access to the submodule 1131 // from the returned pointer. 1132 if (constants_.use_new_agc) { 1133 return public_submodules_->gain_control_for_new_agc.get(); 1134 } 1135 return public_submodules_->gain_control; 1136} 1137 1138HighPassFilter* AudioProcessingImpl::high_pass_filter() const { 1139 // Adding a lock here has no effect as it allows any access to the submodule 1140 // from the returned pointer. 1141 return public_submodules_->high_pass_filter.get(); 1142} 1143 1144LevelEstimator* AudioProcessingImpl::level_estimator() const { 1145 // Adding a lock here has no effect as it allows any access to the submodule 1146 // from the returned pointer. 1147 return public_submodules_->level_estimator.get(); 1148} 1149 1150NoiseSuppression* AudioProcessingImpl::noise_suppression() const { 1151 // Adding a lock here has no effect as it allows any access to the submodule 1152 // from the returned pointer. 1153 return public_submodules_->noise_suppression.get(); 1154} 1155 1156VoiceDetection* AudioProcessingImpl::voice_detection() const { 1157 // Adding a lock here has no effect as it allows any access to the submodule 1158 // from the returned pointer. 1159 return public_submodules_->voice_detection; 1160} 1161 1162bool AudioProcessingImpl::is_data_processed() const { 1163 if (constants_.beamformer_enabled) { 1164 return true; 1165 } 1166 1167 int enabled_count = 0; 1168 for (auto item : private_submodules_->component_list) { 1169 if (item->is_component_enabled()) { 1170 enabled_count++; 1171 } 1172 } 1173 if (public_submodules_->high_pass_filter->is_enabled()) { 1174 enabled_count++; 1175 } 1176 if (public_submodules_->noise_suppression->is_enabled()) { 1177 enabled_count++; 1178 } 1179 if (public_submodules_->level_estimator->is_enabled()) { 1180 enabled_count++; 1181 } 1182 1183 // Data is unchanged if no components are enabled, or if only 1184 // public_submodules_->level_estimator 1185 // or public_submodules_->voice_detection is enabled. 1186 if (enabled_count == 0) { 1187 return false; 1188 } else if (enabled_count == 1) { 1189 if (public_submodules_->level_estimator->is_enabled() || 1190 public_submodules_->voice_detection->is_enabled()) { 1191 return false; 1192 } 1193 } else if (enabled_count == 2) { 1194 if (public_submodules_->level_estimator->is_enabled() && 1195 public_submodules_->voice_detection->is_enabled()) { 1196 return false; 1197 } 1198 } 1199 return true; 1200} 1201 1202bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { 1203 // Check if we've upmixed or downmixed the audio. 1204 return ((formats_.api_format.output_stream().num_channels() != 1205 formats_.api_format.input_stream().num_channels()) || 1206 is_data_processed || capture_.transient_suppressor_enabled); 1207} 1208 1209bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { 1210 return (is_data_processed && 1211 (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == 1212 kSampleRate32kHz || 1213 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == 1214 kSampleRate48kHz)); 1215} 1216 1217bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { 1218 if (!is_data_processed && 1219 !public_submodules_->voice_detection->is_enabled() && 1220 !capture_.transient_suppressor_enabled) { 1221 // Only public_submodules_->level_estimator is enabled. 1222 return false; 1223 } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == 1224 kSampleRate32kHz || 1225 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == 1226 kSampleRate48kHz) { 1227 // Something besides public_submodules_->level_estimator is enabled, and we 1228 // have super-wb. 1229 return true; 1230 } 1231 return false; 1232} 1233 1234bool AudioProcessingImpl::is_rev_processed() const { 1235 return constants_.intelligibility_enabled && 1236 public_submodules_->intelligibility_enhancer->active(); 1237} 1238 1239bool AudioProcessingImpl::render_check_rev_conversion_needed() const { 1240 return rev_conversion_needed(); 1241} 1242 1243bool AudioProcessingImpl::rev_conversion_needed() const { 1244 return (formats_.api_format.reverse_input_stream() != 1245 formats_.api_format.reverse_output_stream()); 1246} 1247 1248void AudioProcessingImpl::InitializeExperimentalAgc() { 1249 if (constants_.use_new_agc) { 1250 if (!private_submodules_->agc_manager.get()) { 1251 private_submodules_->agc_manager.reset(new AgcManagerDirect( 1252 public_submodules_->gain_control, 1253 public_submodules_->gain_control_for_new_agc.get(), 1254 constants_.agc_startup_min_volume)); 1255 } 1256 private_submodules_->agc_manager->Initialize(); 1257 private_submodules_->agc_manager->SetCaptureMuted( 1258 capture_.output_will_be_muted); 1259 } 1260} 1261 1262void AudioProcessingImpl::InitializeTransient() { 1263 if (capture_.transient_suppressor_enabled) { 1264 if (!public_submodules_->transient_suppressor.get()) { 1265 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); 1266 } 1267 public_submodules_->transient_suppressor->Initialize( 1268 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), 1269 capture_nonlocked_.split_rate, 1270 formats_.api_format.output_stream().num_channels()); 1271 } 1272} 1273 1274void AudioProcessingImpl::InitializeBeamformer() { 1275 if (constants_.beamformer_enabled) { 1276 if (!private_submodules_->beamformer) { 1277 private_submodules_->beamformer.reset(new NonlinearBeamformer( 1278 constants_.array_geometry, constants_.target_direction)); 1279 } 1280 private_submodules_->beamformer->Initialize(kChunkSizeMs, 1281 capture_nonlocked_.split_rate); 1282 } 1283} 1284 1285void AudioProcessingImpl::InitializeIntelligibility() { 1286 if (constants_.intelligibility_enabled) { 1287 IntelligibilityEnhancer::Config config; 1288 config.sample_rate_hz = capture_nonlocked_.split_rate; 1289 config.num_capture_channels = capture_.capture_audio->num_channels(); 1290 config.num_render_channels = render_.render_audio->num_channels(); 1291 public_submodules_->intelligibility_enhancer.reset( 1292 new IntelligibilityEnhancer(config)); 1293 } 1294} 1295 1296void AudioProcessingImpl::InitializeHighPassFilter() { 1297 public_submodules_->high_pass_filter->Initialize(num_output_channels(), 1298 proc_sample_rate_hz()); 1299} 1300 1301void AudioProcessingImpl::InitializeNoiseSuppression() { 1302 public_submodules_->noise_suppression->Initialize(num_output_channels(), 1303 proc_sample_rate_hz()); 1304} 1305 1306void AudioProcessingImpl::InitializeLevelEstimator() { 1307 public_submodules_->level_estimator->Initialize(); 1308} 1309 1310void AudioProcessingImpl::MaybeUpdateHistograms() { 1311 static const int kMinDiffDelayMs = 60; 1312 1313 if (echo_cancellation()->is_enabled()) { 1314 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. 1315 // If a stream has echo we know that the echo_cancellation is in process. 1316 if (capture_.stream_delay_jumps == -1 && 1317 echo_cancellation()->stream_has_echo()) { 1318 capture_.stream_delay_jumps = 0; 1319 } 1320 if (capture_.aec_system_delay_jumps == -1 && 1321 echo_cancellation()->stream_has_echo()) { 1322 capture_.aec_system_delay_jumps = 0; 1323 } 1324 1325 // Detect a jump in platform reported system delay and log the difference. 1326 const int diff_stream_delay_ms = 1327 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; 1328 if (diff_stream_delay_ms > kMinDiffDelayMs && 1329 capture_.last_stream_delay_ms != 0) { 1330 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", 1331 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); 1332 if (capture_.stream_delay_jumps == -1) { 1333 capture_.stream_delay_jumps = 0; // Activate counter if needed. 1334 } 1335 capture_.stream_delay_jumps++; 1336 } 1337 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; 1338 1339 // Detect a jump in AEC system delay and log the difference. 1340 const int frames_per_ms = 1341 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); 1342 const int aec_system_delay_ms = 1343 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; 1344 const int diff_aec_system_delay_ms = 1345 aec_system_delay_ms - capture_.last_aec_system_delay_ms; 1346 if (diff_aec_system_delay_ms > kMinDiffDelayMs && 1347 capture_.last_aec_system_delay_ms != 0) { 1348 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", 1349 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, 1350 100); 1351 if (capture_.aec_system_delay_jumps == -1) { 1352 capture_.aec_system_delay_jumps = 0; // Activate counter if needed. 1353 } 1354 capture_.aec_system_delay_jumps++; 1355 } 1356 capture_.last_aec_system_delay_ms = aec_system_delay_ms; 1357 } 1358} 1359 1360void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { 1361 // Run in a single-threaded manner. 1362 rtc::CritScope cs_render(&crit_render_); 1363 rtc::CritScope cs_capture(&crit_capture_); 1364 1365 if (capture_.stream_delay_jumps > -1) { 1366 RTC_HISTOGRAM_ENUMERATION( 1367 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", 1368 capture_.stream_delay_jumps, 51); 1369 } 1370 capture_.stream_delay_jumps = -1; 1371 capture_.last_stream_delay_ms = 0; 1372 1373 if (capture_.aec_system_delay_jumps > -1) { 1374 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", 1375 capture_.aec_system_delay_jumps, 51); 1376 } 1377 capture_.aec_system_delay_jumps = -1; 1378 capture_.last_aec_system_delay_ms = 0; 1379} 1380 1381#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1382int AudioProcessingImpl::WriteMessageToDebugFile( 1383 FileWrapper* debug_file, 1384 rtc::CriticalSection* crit_debug, 1385 ApmDebugDumpThreadState* debug_state) { 1386 int32_t size = debug_state->event_msg->ByteSize(); 1387 if (size <= 0) { 1388 return kUnspecifiedError; 1389 } 1390#if defined(WEBRTC_ARCH_BIG_ENDIAN) 1391// TODO(ajm): Use little-endian "on the wire". For the moment, we can be 1392// pretty safe in assuming little-endian. 1393#endif 1394 1395 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) { 1396 return kUnspecifiedError; 1397 } 1398 1399 { 1400 // Ensure atomic writes of the message. 1401 rtc::CritScope cs_capture(crit_debug); 1402 // Write message preceded by its size. 1403 if (!debug_file->Write(&size, sizeof(int32_t))) { 1404 return kFileError; 1405 } 1406 if (!debug_file->Write(debug_state->event_str.data(), 1407 debug_state->event_str.length())) { 1408 return kFileError; 1409 } 1410 } 1411 1412 debug_state->event_msg->Clear(); 1413 1414 return kNoError; 1415} 1416 1417int AudioProcessingImpl::WriteInitMessage() { 1418 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT); 1419 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init(); 1420 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz()); 1421 1422 msg->set_num_input_channels( 1423 formats_.api_format.input_stream().num_channels()); 1424 msg->set_num_output_channels( 1425 formats_.api_format.output_stream().num_channels()); 1426 msg->set_num_reverse_channels( 1427 formats_.api_format.reverse_input_stream().num_channels()); 1428 msg->set_reverse_sample_rate( 1429 formats_.api_format.reverse_input_stream().sample_rate_hz()); 1430 msg->set_output_sample_rate( 1431 formats_.api_format.output_stream().sample_rate_hz()); 1432 // TODO(ekmeyerson): Add reverse output fields to 1433 // debug_dump_.capture.event_msg. 1434 1435 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 1436 &crit_debug_, &debug_dump_.capture)); 1437 return kNoError; 1438} 1439 1440int AudioProcessingImpl::WriteConfigMessage(bool forced) { 1441 audioproc::Config config; 1442 1443 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled()); 1444 config.set_aec_delay_agnostic_enabled( 1445 public_submodules_->echo_cancellation->is_delay_agnostic_enabled()); 1446 config.set_aec_drift_compensation_enabled( 1447 public_submodules_->echo_cancellation->is_drift_compensation_enabled()); 1448 config.set_aec_extended_filter_enabled( 1449 public_submodules_->echo_cancellation->is_extended_filter_enabled()); 1450 config.set_aec_suppression_level(static_cast<int>( 1451 public_submodules_->echo_cancellation->suppression_level())); 1452 1453 config.set_aecm_enabled( 1454 public_submodules_->echo_control_mobile->is_enabled()); 1455 config.set_aecm_comfort_noise_enabled( 1456 public_submodules_->echo_control_mobile->is_comfort_noise_enabled()); 1457 config.set_aecm_routing_mode(static_cast<int>( 1458 public_submodules_->echo_control_mobile->routing_mode())); 1459 1460 config.set_agc_enabled(public_submodules_->gain_control->is_enabled()); 1461 config.set_agc_mode( 1462 static_cast<int>(public_submodules_->gain_control->mode())); 1463 config.set_agc_limiter_enabled( 1464 public_submodules_->gain_control->is_limiter_enabled()); 1465 config.set_noise_robust_agc_enabled(constants_.use_new_agc); 1466 1467 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled()); 1468 1469 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled()); 1470 config.set_ns_level( 1471 static_cast<int>(public_submodules_->noise_suppression->level())); 1472 1473 config.set_transient_suppression_enabled( 1474 capture_.transient_suppressor_enabled); 1475 1476 std::string serialized_config = config.SerializeAsString(); 1477 if (!forced && 1478 debug_dump_.capture.last_serialized_config == serialized_config) { 1479 return kNoError; 1480 } 1481 1482 debug_dump_.capture.last_serialized_config = serialized_config; 1483 1484 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG); 1485 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config); 1486 1487 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), 1488 &crit_debug_, &debug_dump_.capture)); 1489 return kNoError; 1490} 1491#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1492 1493} // namespace webrtc 1494