audio_input_controller.cc revision 6e8cce623b6e4fe0c9e4af605d675dd9d0338c38
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "media/audio/audio_input_controller.h" 6 7#include "base/bind.h" 8#include "base/strings/stringprintf.h" 9#include "base/threading/thread_restrictions.h" 10#include "base/time/time.h" 11#include "media/base/limits.h" 12#include "media/base/scoped_histogram_timer.h" 13#include "media/base/user_input_monitor.h" 14 15using base::TimeDelta; 16 17namespace { 18const int kMaxInputChannels = 3; 19 20// TODO(henrika): remove usage of timers and add support for proper 21// notification of when the input device is removed. This was originally added 22// to resolve http://crbug.com/79936 for Windows platforms. This then caused 23// breakage (very hard to repro bugs!) on other platforms: See 24// http://crbug.com/226327 and http://crbug.com/230972. 25// See also that the timer has been disabled on Mac now due to 26// crbug.com/357501. 27const int kTimerResetIntervalSeconds = 1; 28// We have received reports that the timer can be too trigger happy on some 29// Mac devices and the initial timer interval has therefore been increased 30// from 1 second to 5 seconds. 31const int kTimerInitialIntervalSeconds = 5; 32 33#if defined(AUDIO_POWER_MONITORING) 34// Time constant for AudioPowerMonitor. 35// The utilized smoothing factor (alpha) in the exponential filter is given 36// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time 37// constant given by |kPowerMeasurementTimeConstantMilliseconds|. 38// Example: fs=44100, ts=10e-3 => alpha~0.022420 39// fs=44100, ts=20e-3 => alpha~0.165903 40// A large smoothing factor corresponds to a faster filter response to input 41// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input 42// and y(n) is the output. 43const int kPowerMeasurementTimeConstantMilliseconds = 10; 44 45// Time in seconds between two successive measurements of audio power levels. 46const int kPowerMonitorLogIntervalSeconds = 5; 47#endif 48} 49 50// Used to log the result of capture startup. 51// This was previously logged as a boolean with only the no callback and OK 52// options. The enum order is kept to ensure backwards compatibility. 53// Elements in this enum should not be deleted or rearranged; the only 54// permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX 55// and update CAPTURE_STARTUP_RESULT_MAX. 56enum CaptureStartupResult { 57 CAPTURE_STARTUP_NO_DATA_CALLBACK = 0, 58 CAPTURE_STARTUP_OK = 1, 59 CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2, 60 CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3, 61 CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED 62}; 63 64void LogCaptureStartupResult(CaptureStartupResult result) { 65 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess", 66 result, 67 CAPTURE_STARTUP_RESULT_MAX + 1); 68 69} 70 71namespace media { 72 73// static 74AudioInputController::Factory* AudioInputController::factory_ = NULL; 75 76AudioInputController::AudioInputController(EventHandler* handler, 77 SyncWriter* sync_writer, 78 UserInputMonitor* user_input_monitor) 79 : creator_task_runner_(base::MessageLoopProxy::current()), 80 handler_(handler), 81 stream_(NULL), 82 data_is_active_(false), 83 state_(CLOSED), 84 sync_writer_(sync_writer), 85 max_volume_(0.0), 86 user_input_monitor_(user_input_monitor), 87#if defined(AUDIO_POWER_MONITORING) 88 silence_state_(SILENCE_STATE_NO_MEASUREMENT), 89#endif 90 prev_key_down_count_(0) { 91 DCHECK(creator_task_runner_.get()); 92} 93 94AudioInputController::~AudioInputController() { 95 DCHECK_EQ(state_, CLOSED); 96} 97 98// static 99scoped_refptr<AudioInputController> AudioInputController::Create( 100 AudioManager* audio_manager, 101 EventHandler* event_handler, 102 const AudioParameters& params, 103 const std::string& device_id, 104 UserInputMonitor* user_input_monitor) { 105 DCHECK(audio_manager); 106 107 if (!params.IsValid() || (params.channels() > kMaxInputChannels)) 108 return NULL; 109 110 if (factory_) { 111 return factory_->Create( 112 audio_manager, event_handler, params, user_input_monitor); 113 } 114 scoped_refptr<AudioInputController> controller( 115 new AudioInputController(event_handler, NULL, user_input_monitor)); 116 117 controller->task_runner_ = audio_manager->GetTaskRunner(); 118 119 // Create and open a new audio input stream from the existing 120 // audio-device thread. 121 if (!controller->task_runner_->PostTask(FROM_HERE, 122 base::Bind(&AudioInputController::DoCreate, controller, 123 base::Unretained(audio_manager), params, device_id))) { 124 controller = NULL; 125 } 126 127 return controller; 128} 129 130// static 131scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency( 132 AudioManager* audio_manager, 133 EventHandler* event_handler, 134 const AudioParameters& params, 135 const std::string& device_id, 136 SyncWriter* sync_writer, 137 UserInputMonitor* user_input_monitor) { 138 DCHECK(audio_manager); 139 DCHECK(sync_writer); 140 141 if (!params.IsValid() || (params.channels() > kMaxInputChannels)) 142 return NULL; 143 144 // Create the AudioInputController object and ensure that it runs on 145 // the audio-manager thread. 146 scoped_refptr<AudioInputController> controller( 147 new AudioInputController(event_handler, sync_writer, user_input_monitor)); 148 controller->task_runner_ = audio_manager->GetTaskRunner(); 149 150 // Create and open a new audio input stream from the existing 151 // audio-device thread. Use the provided audio-input device. 152 if (!controller->task_runner_->PostTask(FROM_HERE, 153 base::Bind(&AudioInputController::DoCreate, controller, 154 base::Unretained(audio_manager), params, device_id))) { 155 controller = NULL; 156 } 157 158 return controller; 159} 160 161// static 162scoped_refptr<AudioInputController> AudioInputController::CreateForStream( 163 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, 164 EventHandler* event_handler, 165 AudioInputStream* stream, 166 SyncWriter* sync_writer, 167 UserInputMonitor* user_input_monitor) { 168 DCHECK(sync_writer); 169 DCHECK(stream); 170 171 // Create the AudioInputController object and ensure that it runs on 172 // the audio-manager thread. 173 scoped_refptr<AudioInputController> controller( 174 new AudioInputController(event_handler, sync_writer, user_input_monitor)); 175 controller->task_runner_ = task_runner; 176 177 // TODO(miu): See TODO at top of file. Until that's resolved, we need to 178 // disable the error auto-detection here (since the audio mirroring 179 // implementation will reliably report error and close events). Note, of 180 // course, that we're assuming CreateForStream() has been called for the audio 181 // mirroring use case only. 182 if (!controller->task_runner_->PostTask( 183 FROM_HERE, 184 base::Bind(&AudioInputController::DoCreateForStream, 185 controller, 186 stream))) { 187 controller = NULL; 188 } 189 190 return controller; 191} 192 193void AudioInputController::Record() { 194 task_runner_->PostTask(FROM_HERE, base::Bind( 195 &AudioInputController::DoRecord, this)); 196} 197 198void AudioInputController::Close(const base::Closure& closed_task) { 199 DCHECK(!closed_task.is_null()); 200 DCHECK(creator_task_runner_->BelongsToCurrentThread()); 201 202 task_runner_->PostTaskAndReply( 203 FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task); 204} 205 206void AudioInputController::SetVolume(double volume) { 207 task_runner_->PostTask(FROM_HERE, base::Bind( 208 &AudioInputController::DoSetVolume, this, volume)); 209} 210 211void AudioInputController::SetAutomaticGainControl(bool enabled) { 212 task_runner_->PostTask(FROM_HERE, base::Bind( 213 &AudioInputController::DoSetAutomaticGainControl, this, enabled)); 214} 215 216void AudioInputController::DoCreate(AudioManager* audio_manager, 217 const AudioParameters& params, 218 const std::string& device_id) { 219 DCHECK(task_runner_->BelongsToCurrentThread()); 220 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime"); 221 222#if defined(AUDIO_POWER_MONITORING) 223 // Create the audio (power) level meter given the provided audio parameters. 224 // An AudioBus is also needed to wrap the raw data buffer from the native 225 // layer to match AudioPowerMonitor::Scan(). 226 // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155. 227 audio_level_.reset(new media::AudioPowerMonitor( 228 params.sample_rate(), 229 TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds))); 230 audio_params_ = params; 231 silence_state_ = SILENCE_STATE_NO_MEASUREMENT; 232#endif 233 234 // TODO(miu): See TODO at top of file. Until that's resolved, assume all 235 // platform audio input requires the |no_data_timer_| be used to auto-detect 236 // errors. In reality, probably only Windows needs to be treated as 237 // unreliable here. 238 DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id)); 239} 240 241void AudioInputController::DoCreateForStream( 242 AudioInputStream* stream_to_control) { 243 DCHECK(task_runner_->BelongsToCurrentThread()); 244 245 DCHECK(!stream_); 246 stream_ = stream_to_control; 247 248 if (!stream_) { 249 if (handler_) 250 handler_->OnError(this, STREAM_CREATE_ERROR); 251 LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED); 252 return; 253 } 254 255 if (stream_ && !stream_->Open()) { 256 stream_->Close(); 257 stream_ = NULL; 258 if (handler_) 259 handler_->OnError(this, STREAM_OPEN_ERROR); 260 LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED); 261 return; 262 } 263 264 DCHECK(!no_data_timer_.get()); 265 266 // Create the data timer which will call FirstCheckForNoData(). The timer 267 // is started in DoRecord() and restarted in each DoCheckForNoData() 268 // callback. 269 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered 270 // from the timer must be ignored by the EventHandler. 271 // TODO(henrika): remove usage of timer when it has been verified on Canary 272 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and 273 // everything that is tied to it. crbug.com/357569. 274 no_data_timer_.reset(new base::Timer( 275 FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds), 276 base::Bind(&AudioInputController::FirstCheckForNoData, 277 base::Unretained(this)), false)); 278 279 state_ = CREATED; 280 if (handler_) 281 handler_->OnCreated(this); 282 283 if (user_input_monitor_) { 284 user_input_monitor_->EnableKeyPressMonitoring(); 285 prev_key_down_count_ = user_input_monitor_->GetKeyPressCount(); 286 } 287} 288 289void AudioInputController::DoRecord() { 290 DCHECK(task_runner_->BelongsToCurrentThread()); 291 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime"); 292 293 if (state_ != CREATED) 294 return; 295 296 { 297 base::AutoLock auto_lock(lock_); 298 state_ = RECORDING; 299 } 300 301 if (no_data_timer_) { 302 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed, 303 // a callback to FirstCheckForNoData() is made. 304 no_data_timer_->Reset(); 305 } 306 307 stream_->Start(this); 308 if (handler_) 309 handler_->OnRecording(this); 310} 311 312void AudioInputController::DoClose() { 313 DCHECK(task_runner_->BelongsToCurrentThread()); 314 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime"); 315 316 if (state_ == CLOSED) 317 return; 318 319 // Delete the timer on the same thread that created it. 320 no_data_timer_.reset(); 321 322 DoStopCloseAndClearStream(); 323 SetDataIsActive(false); 324 325 if (SharedMemoryAndSyncSocketMode()) 326 sync_writer_->Close(); 327 328 if (user_input_monitor_) 329 user_input_monitor_->DisableKeyPressMonitoring(); 330 331#if defined(AUDIO_POWER_MONITORING) 332 // Send UMA stats if we have enabled power monitoring. 333 if (audio_level_) { 334 LogSilenceState(silence_state_); 335 } 336#endif 337 338 state_ = CLOSED; 339} 340 341void AudioInputController::DoReportError() { 342 DCHECK(task_runner_->BelongsToCurrentThread()); 343 if (handler_) 344 handler_->OnError(this, STREAM_ERROR); 345} 346 347void AudioInputController::DoSetVolume(double volume) { 348 DCHECK(task_runner_->BelongsToCurrentThread()); 349 DCHECK_GE(volume, 0); 350 DCHECK_LE(volume, 1.0); 351 352 if (state_ != CREATED && state_ != RECORDING) 353 return; 354 355 // Only ask for the maximum volume at first call and use cached value 356 // for remaining function calls. 357 if (!max_volume_) { 358 max_volume_ = stream_->GetMaxVolume(); 359 } 360 361 if (max_volume_ == 0.0) { 362 DLOG(WARNING) << "Failed to access input volume control"; 363 return; 364 } 365 366 // Set the stream volume and scale to a range matched to the platform. 367 stream_->SetVolume(max_volume_ * volume); 368} 369 370void AudioInputController::DoSetAutomaticGainControl(bool enabled) { 371 DCHECK(task_runner_->BelongsToCurrentThread()); 372 DCHECK_NE(state_, RECORDING); 373 374 // Ensure that the AGC state only can be modified before streaming starts. 375 if (state_ != CREATED) 376 return; 377 378 stream_->SetAutomaticGainControl(enabled); 379} 380 381void AudioInputController::FirstCheckForNoData() { 382 DCHECK(task_runner_->BelongsToCurrentThread()); 383 LogCaptureStartupResult(GetDataIsActive() ? 384 CAPTURE_STARTUP_OK : 385 CAPTURE_STARTUP_NO_DATA_CALLBACK); 386 DoCheckForNoData(); 387} 388 389void AudioInputController::DoCheckForNoData() { 390 DCHECK(task_runner_->BelongsToCurrentThread()); 391 392 if (!GetDataIsActive()) { 393 // The data-is-active marker will be false only if it has been more than 394 // one second since a data packet was recorded. This can happen if a 395 // capture device has been removed or disabled. 396 if (handler_) 397 handler_->OnError(this, NO_DATA_ERROR); 398 } 399 400 // Mark data as non-active. The flag will be re-enabled in OnData() each 401 // time a data packet is received. Hence, under normal conditions, the 402 // flag will only be disabled during a very short period. 403 SetDataIsActive(false); 404 405 // Restart the timer to ensure that we check the flag again in 406 // |kTimerResetIntervalSeconds|. 407 no_data_timer_->Start( 408 FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds), 409 base::Bind(&AudioInputController::DoCheckForNoData, 410 base::Unretained(this))); 411} 412 413void AudioInputController::OnData(AudioInputStream* stream, 414 const AudioBus* source, 415 uint32 hardware_delay_bytes, 416 double volume) { 417 // Mark data as active to ensure that the periodic calls to 418 // DoCheckForNoData() does not report an error to the event handler. 419 SetDataIsActive(true); 420 421 { 422 base::AutoLock auto_lock(lock_); 423 if (state_ != RECORDING) 424 return; 425 } 426 427 bool key_pressed = false; 428 if (user_input_monitor_) { 429 size_t current_count = user_input_monitor_->GetKeyPressCount(); 430 key_pressed = current_count != prev_key_down_count_; 431 prev_key_down_count_ = current_count; 432 DVLOG_IF(6, key_pressed) << "Detected keypress."; 433 } 434 435 // Use SharedMemory and SyncSocket if the client has created a SyncWriter. 436 // Used by all low-latency clients except WebSpeech. 437 if (SharedMemoryAndSyncSocketMode()) { 438 sync_writer_->Write(source, volume, key_pressed); 439 sync_writer_->UpdateRecordedBytes(hardware_delay_bytes); 440 441#if defined(AUDIO_POWER_MONITORING) 442 // Only do power-level measurements if an AudioPowerMonitor object has 443 // been created. Done in DoCreate() but not DoCreateForStream(), hence 444 // logging will mainly be done for WebRTC and WebSpeech clients. 445 if (!audio_level_) 446 return; 447 448 // Perform periodic audio (power) level measurements. 449 if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() > 450 kPowerMonitorLogIntervalSeconds) { 451 // Wrap data into an AudioBus to match AudioPowerMonitor::Scan. 452 // TODO(henrika): remove this section when capture side uses AudioBus. 453 // See http://crbug.com/375155 for details. 454 audio_level_->Scan(*source, source->frames()); 455 456 // Get current average power level and add it to the log. 457 // Possible range is given by [-inf, 0] dBFS. 458 std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip(); 459 460 // Use event handler on the audio thread to relay a message to the ARIH 461 // in content which does the actual logging on the IO thread. 462 task_runner_->PostTask( 463 FROM_HERE, 464 base::Bind( 465 &AudioInputController::DoLogAudioLevel, this, result.first)); 466 467 last_audio_level_log_time_ = base::TimeTicks::Now(); 468 469 // Reset the average power level (since we don't log continuously). 470 audio_level_->Reset(); 471 } 472#endif 473 return; 474 } 475 476 // TODO(henrika): Investigate if we can avoid the extra copy here. 477 // (see http://crbug.com/249316 for details). AFAIK, this scope is only 478 // active for WebSpeech clients. 479 scoped_ptr<AudioBus> audio_data = 480 AudioBus::Create(source->channels(), source->frames()); 481 source->CopyTo(audio_data.get()); 482 483 // Ownership of the audio buffer will be with the callback until it is run, 484 // when ownership is passed to the callback function. 485 task_runner_->PostTask( 486 FROM_HERE, 487 base::Bind( 488 &AudioInputController::DoOnData, this, base::Passed(&audio_data))); 489} 490 491void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) { 492 DCHECK(task_runner_->BelongsToCurrentThread()); 493 if (handler_) 494 handler_->OnData(this, data.get()); 495} 496 497void AudioInputController::DoLogAudioLevel(float level_dbfs) { 498#if defined(AUDIO_POWER_MONITORING) 499 DCHECK(task_runner_->BelongsToCurrentThread()); 500 if (!handler_) 501 return; 502 503 std::string log_string = base::StringPrintf( 504 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs); 505 static const float kSilenceThresholdDBFS = -72.24719896f; 506 if (level_dbfs < kSilenceThresholdDBFS) { 507 log_string += " <=> no audio input!"; 508 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) 509 silence_state_ = SILENCE_STATE_ONLY_SILENCE; 510 else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) 511 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE; 512 else 513 DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE || 514 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE); 515 } else { 516 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) 517 silence_state_ = SILENCE_STATE_ONLY_AUDIO; 518 else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) 519 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE; 520 else 521 DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO || 522 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE); 523 } 524 525 handler_->OnLog(this, log_string); 526#endif 527} 528 529void AudioInputController::OnError(AudioInputStream* stream) { 530 // Handle error on the audio-manager thread. 531 task_runner_->PostTask(FROM_HERE, base::Bind( 532 &AudioInputController::DoReportError, this)); 533} 534 535void AudioInputController::DoStopCloseAndClearStream() { 536 DCHECK(task_runner_->BelongsToCurrentThread()); 537 538 // Allow calling unconditionally and bail if we don't have a stream to close. 539 if (stream_ != NULL) { 540 stream_->Stop(); 541 stream_->Close(); 542 stream_ = NULL; 543 } 544 545 // The event handler should not be touched after the stream has been closed. 546 handler_ = NULL; 547} 548 549void AudioInputController::SetDataIsActive(bool enabled) { 550 base::subtle::Release_Store(&data_is_active_, enabled); 551} 552 553bool AudioInputController::GetDataIsActive() { 554 return (base::subtle::Acquire_Load(&data_is_active_) != false); 555} 556 557#if defined(AUDIO_POWER_MONITORING) 558void AudioInputController::LogSilenceState(SilenceState value) { 559 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport", 560 value, 561 SILENCE_STATE_MAX + 1); 562} 563#endif 564 565} // namespace media 566