audio_input_controller.cc revision cedac228d2dd51db4b79ea1e72c7f249408ee061
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "media/audio/audio_input_controller.h" 6 7#include "base/bind.h" 8#include "base/strings/stringprintf.h" 9#include "base/threading/thread_restrictions.h" 10#include "base/time/time.h" 11#include "media/base/limits.h" 12#include "media/base/scoped_histogram_timer.h" 13#include "media/base/user_input_monitor.h" 14 15using base::TimeDelta; 16 17namespace { 18const int kMaxInputChannels = 3; 19 20// TODO(henrika): remove usage of timers and add support for proper 21// notification of when the input device is removed. This was originally added 22// to resolve http://crbug.com/79936 for Windows platforms. This then caused 23// breakage (very hard to repro bugs!) on other platforms: See 24// http://crbug.com/226327 and http://crbug.com/230972. 25// See also that the timer has been disabled on Mac now due to 26// crbug.com/357501. 27const int kTimerResetIntervalSeconds = 1; 28// We have received reports that the timer can be too trigger happy on some 29// Mac devices and the initial timer interval has therefore been increased 30// from 1 second to 5 seconds. 31const int kTimerInitialIntervalSeconds = 5; 32 33#if defined(AUDIO_POWER_MONITORING) 34// Time constant for AudioPowerMonitor. 35// The utilized smoothing factor (alpha) in the exponential filter is given 36// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time 37// constant given by |kPowerMeasurementTimeConstantMilliseconds|. 38// Example: fs=44100, ts=10e-3 => alpha~0.022420 39// fs=44100, ts=20e-3 => alpha~0.165903 40// A large smoothing factor corresponds to a faster filter response to input 41// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input 42// and y(n) is the output. 43const int kPowerMeasurementTimeConstantMilliseconds = 10; 44 45// Time in seconds between two successive measurements of audio power levels. 46const int kPowerMonitorLogIntervalSeconds = 5; 47#endif 48} 49 50namespace media { 51 52// static 53AudioInputController::Factory* AudioInputController::factory_ = NULL; 54 55AudioInputController::AudioInputController(EventHandler* handler, 56 SyncWriter* sync_writer, 57 UserInputMonitor* user_input_monitor) 58 : creator_task_runner_(base::MessageLoopProxy::current()), 59 handler_(handler), 60 stream_(NULL), 61 data_is_active_(false), 62 state_(CLOSED), 63 sync_writer_(sync_writer), 64 max_volume_(0.0), 65 user_input_monitor_(user_input_monitor), 66 prev_key_down_count_(0) { 67 DCHECK(creator_task_runner_.get()); 68} 69 70AudioInputController::~AudioInputController() { 71 DCHECK_EQ(state_, CLOSED); 72} 73 74// static 75scoped_refptr<AudioInputController> AudioInputController::Create( 76 AudioManager* audio_manager, 77 EventHandler* event_handler, 78 const AudioParameters& params, 79 const std::string& device_id, 80 UserInputMonitor* user_input_monitor) { 81 DCHECK(audio_manager); 82 83 if (!params.IsValid() || (params.channels() > kMaxInputChannels)) 84 return NULL; 85 86 if (factory_) { 87 return factory_->Create( 88 audio_manager, event_handler, params, user_input_monitor); 89 } 90 scoped_refptr<AudioInputController> controller( 91 new AudioInputController(event_handler, NULL, user_input_monitor)); 92 93 controller->task_runner_ = audio_manager->GetTaskRunner(); 94 95 // Create and open a new audio input stream from the existing 96 // audio-device thread. 97 if (!controller->task_runner_->PostTask(FROM_HERE, 98 base::Bind(&AudioInputController::DoCreate, controller, 99 base::Unretained(audio_manager), params, device_id))) { 100 controller = NULL; 101 } 102 103 return controller; 104} 105 106// static 107scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency( 108 AudioManager* audio_manager, 109 EventHandler* event_handler, 110 const AudioParameters& params, 111 const std::string& device_id, 112 SyncWriter* sync_writer, 113 UserInputMonitor* user_input_monitor) { 114 DCHECK(audio_manager); 115 DCHECK(sync_writer); 116 117 if (!params.IsValid() || (params.channels() > kMaxInputChannels)) 118 return NULL; 119 120 // Create the AudioInputController object and ensure that it runs on 121 // the audio-manager thread. 122 scoped_refptr<AudioInputController> controller( 123 new AudioInputController(event_handler, sync_writer, user_input_monitor)); 124 controller->task_runner_ = audio_manager->GetTaskRunner(); 125 126 // Create and open a new audio input stream from the existing 127 // audio-device thread. Use the provided audio-input device. 128 if (!controller->task_runner_->PostTask(FROM_HERE, 129 base::Bind(&AudioInputController::DoCreate, controller, 130 base::Unretained(audio_manager), params, device_id))) { 131 controller = NULL; 132 } 133 134 return controller; 135} 136 137// static 138scoped_refptr<AudioInputController> AudioInputController::CreateForStream( 139 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, 140 EventHandler* event_handler, 141 AudioInputStream* stream, 142 SyncWriter* sync_writer, 143 UserInputMonitor* user_input_monitor) { 144 DCHECK(sync_writer); 145 DCHECK(stream); 146 147 // Create the AudioInputController object and ensure that it runs on 148 // the audio-manager thread. 149 scoped_refptr<AudioInputController> controller( 150 new AudioInputController(event_handler, sync_writer, user_input_monitor)); 151 controller->task_runner_ = task_runner; 152 153 // TODO(miu): See TODO at top of file. Until that's resolved, we need to 154 // disable the error auto-detection here (since the audio mirroring 155 // implementation will reliably report error and close events). Note, of 156 // course, that we're assuming CreateForStream() has been called for the audio 157 // mirroring use case only. 158 if (!controller->task_runner_->PostTask( 159 FROM_HERE, 160 base::Bind(&AudioInputController::DoCreateForStream, controller, 161 stream, false))) { 162 controller = NULL; 163 } 164 165 return controller; 166} 167 168void AudioInputController::Record() { 169 task_runner_->PostTask(FROM_HERE, base::Bind( 170 &AudioInputController::DoRecord, this)); 171} 172 173void AudioInputController::Close(const base::Closure& closed_task) { 174 DCHECK(!closed_task.is_null()); 175 DCHECK(creator_task_runner_->BelongsToCurrentThread()); 176 177 task_runner_->PostTaskAndReply( 178 FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task); 179} 180 181void AudioInputController::SetVolume(double volume) { 182 task_runner_->PostTask(FROM_HERE, base::Bind( 183 &AudioInputController::DoSetVolume, this, volume)); 184} 185 186void AudioInputController::SetAutomaticGainControl(bool enabled) { 187 task_runner_->PostTask(FROM_HERE, base::Bind( 188 &AudioInputController::DoSetAutomaticGainControl, this, enabled)); 189} 190 191void AudioInputController::DoCreate(AudioManager* audio_manager, 192 const AudioParameters& params, 193 const std::string& device_id) { 194 DCHECK(task_runner_->BelongsToCurrentThread()); 195 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime"); 196 197#if defined(AUDIO_POWER_MONITORING) 198 // Create the audio (power) level meter given the provided audio parameters. 199 // An AudioBus is also needed to wrap the raw data buffer from the native 200 // layer to match AudioPowerMonitor::Scan(). 201 // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155. 202 audio_level_.reset(new media::AudioPowerMonitor( 203 params.sample_rate(), 204 TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds))); 205 audio_bus_ = AudioBus::Create(params); 206 audio_params_ = params; 207#endif 208 209 // TODO(miu): See TODO at top of file. Until that's resolved, assume all 210 // platform audio input requires the |no_data_timer_| be used to auto-detect 211 // errors. In reality, probably only Windows needs to be treated as 212 // unreliable here. 213 DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id), 214 true); 215} 216 217void AudioInputController::DoCreateForStream( 218 AudioInputStream* stream_to_control, bool enable_nodata_timer) { 219 DCHECK(task_runner_->BelongsToCurrentThread()); 220 221 DCHECK(!stream_); 222 stream_ = stream_to_control; 223 224 if (!stream_) { 225 if (handler_) 226 handler_->OnError(this, STREAM_CREATE_ERROR); 227 return; 228 } 229 230 if (stream_ && !stream_->Open()) { 231 stream_->Close(); 232 stream_ = NULL; 233 if (handler_) 234 handler_->OnError(this, STREAM_OPEN_ERROR); 235 return; 236 } 237 238 DCHECK(!no_data_timer_.get()); 239 240 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered 241 // from the timer must be ignored by the EventHandler. 242 // TODO(henrika): remove usage of timer when it has been verified on Canary 243 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and 244 // everything that is tied to it. crbug.com/357569. 245 enable_nodata_timer = true; 246 247 if (enable_nodata_timer) { 248 // Create the data timer which will call FirstCheckForNoData(). The timer 249 // is started in DoRecord() and restarted in each DoCheckForNoData() 250 // callback. 251 no_data_timer_.reset(new base::Timer( 252 FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds), 253 base::Bind(&AudioInputController::FirstCheckForNoData, 254 base::Unretained(this)), false)); 255 } else { 256 DVLOG(1) << "Disabled: timer check for no data."; 257 } 258 259 state_ = CREATED; 260 if (handler_) 261 handler_->OnCreated(this); 262 263 if (user_input_monitor_) { 264 user_input_monitor_->EnableKeyPressMonitoring(); 265 prev_key_down_count_ = user_input_monitor_->GetKeyPressCount(); 266 } 267} 268 269void AudioInputController::DoRecord() { 270 DCHECK(task_runner_->BelongsToCurrentThread()); 271 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime"); 272 273 if (state_ != CREATED) 274 return; 275 276 { 277 base::AutoLock auto_lock(lock_); 278 state_ = RECORDING; 279 } 280 281 if (no_data_timer_) { 282 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed, 283 // a callback to FirstCheckForNoData() is made. 284 no_data_timer_->Reset(); 285 } 286 287 stream_->Start(this); 288 if (handler_) 289 handler_->OnRecording(this); 290} 291 292void AudioInputController::DoClose() { 293 DCHECK(task_runner_->BelongsToCurrentThread()); 294 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime"); 295 296 if (state_ == CLOSED) 297 return; 298 299 // Delete the timer on the same thread that created it. 300 no_data_timer_.reset(); 301 302 DoStopCloseAndClearStream(); 303 SetDataIsActive(false); 304 305 if (SharedMemoryAndSyncSocketMode()) 306 sync_writer_->Close(); 307 308 if (user_input_monitor_) 309 user_input_monitor_->DisableKeyPressMonitoring(); 310 311 state_ = CLOSED; 312} 313 314void AudioInputController::DoReportError() { 315 DCHECK(task_runner_->BelongsToCurrentThread()); 316 if (handler_) 317 handler_->OnError(this, STREAM_ERROR); 318} 319 320void AudioInputController::DoSetVolume(double volume) { 321 DCHECK(task_runner_->BelongsToCurrentThread()); 322 DCHECK_GE(volume, 0); 323 DCHECK_LE(volume, 1.0); 324 325 if (state_ != CREATED && state_ != RECORDING) 326 return; 327 328 // Only ask for the maximum volume at first call and use cached value 329 // for remaining function calls. 330 if (!max_volume_) { 331 max_volume_ = stream_->GetMaxVolume(); 332 } 333 334 if (max_volume_ == 0.0) { 335 DLOG(WARNING) << "Failed to access input volume control"; 336 return; 337 } 338 339 // Set the stream volume and scale to a range matched to the platform. 340 stream_->SetVolume(max_volume_ * volume); 341} 342 343void AudioInputController::DoSetAutomaticGainControl(bool enabled) { 344 DCHECK(task_runner_->BelongsToCurrentThread()); 345 DCHECK_NE(state_, RECORDING); 346 347 // Ensure that the AGC state only can be modified before streaming starts. 348 if (state_ != CREATED) 349 return; 350 351 stream_->SetAutomaticGainControl(enabled); 352} 353 354void AudioInputController::FirstCheckForNoData() { 355 DCHECK(task_runner_->BelongsToCurrentThread()); 356 UMA_HISTOGRAM_BOOLEAN("Media.AudioInputControllerCaptureStartupSuccess", 357 GetDataIsActive()); 358 DoCheckForNoData(); 359} 360 361void AudioInputController::DoCheckForNoData() { 362 DCHECK(task_runner_->BelongsToCurrentThread()); 363 364 if (!GetDataIsActive()) { 365 // The data-is-active marker will be false only if it has been more than 366 // one second since a data packet was recorded. This can happen if a 367 // capture device has been removed or disabled. 368 if (handler_) 369 handler_->OnError(this, NO_DATA_ERROR); 370 } 371 372 // Mark data as non-active. The flag will be re-enabled in OnData() each 373 // time a data packet is received. Hence, under normal conditions, the 374 // flag will only be disabled during a very short period. 375 SetDataIsActive(false); 376 377 // Restart the timer to ensure that we check the flag again in 378 // |kTimerResetIntervalSeconds|. 379 no_data_timer_->Start( 380 FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds), 381 base::Bind(&AudioInputController::DoCheckForNoData, 382 base::Unretained(this))); 383} 384 385void AudioInputController::OnData(AudioInputStream* stream, 386 const uint8* data, 387 uint32 size, 388 uint32 hardware_delay_bytes, 389 double volume) { 390 // Mark data as active to ensure that the periodic calls to 391 // DoCheckForNoData() does not report an error to the event handler. 392 SetDataIsActive(true); 393 394 { 395 base::AutoLock auto_lock(lock_); 396 if (state_ != RECORDING) 397 return; 398 } 399 400 bool key_pressed = false; 401 if (user_input_monitor_) { 402 size_t current_count = user_input_monitor_->GetKeyPressCount(); 403 key_pressed = current_count != prev_key_down_count_; 404 prev_key_down_count_ = current_count; 405 DVLOG_IF(6, key_pressed) << "Detected keypress."; 406 } 407 408 // Use SharedMemory and SyncSocket if the client has created a SyncWriter. 409 // Used by all low-latency clients except WebSpeech. 410 if (SharedMemoryAndSyncSocketMode()) { 411 sync_writer_->Write(data, size, volume, key_pressed); 412 sync_writer_->UpdateRecordedBytes(hardware_delay_bytes); 413 414#if defined(AUDIO_POWER_MONITORING) 415 // Only do power-level measurements if an AudioPowerMonitor object has 416 // been created. Done in DoCreate() but not DoCreateForStream(), hence 417 // logging will mainly be done for WebRTC and WebSpeech clients. 418 if (!audio_level_) 419 return; 420 421 // Perform periodic audio (power) level measurements. 422 if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() > 423 kPowerMonitorLogIntervalSeconds) { 424 // Wrap data into an AudioBus to match AudioPowerMonitor::Scan. 425 // TODO(henrika): remove this section when capture side uses AudioBus. 426 // See http://crbug.com/375155 for details. 427 audio_bus_->FromInterleaved( 428 data, audio_bus_->frames(), audio_params_.bits_per_sample() / 8); 429 audio_level_->Scan(*audio_bus_, audio_bus_->frames()); 430 431 // Get current average power level and add it to the log. 432 // Possible range is given by [-inf, 0] dBFS. 433 std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip(); 434 435 // Use event handler on the audio thread to relay a message to the ARIH 436 // in content which does the actual logging on the IO thread. 437 task_runner_->PostTask( 438 FROM_HERE, 439 base::Bind( 440 &AudioInputController::DoLogAudioLevel, this, result.first)); 441 442 last_audio_level_log_time_ = base::TimeTicks::Now(); 443 444 // Reset the average power level (since we don't log continuously). 445 audio_level_->Reset(); 446 } 447#endif 448 449 return; 450 } 451 452 // TODO(henrika): Investigate if we can avoid the extra copy here. 453 // (see http://crbug.com/249316 for details). AFAIK, this scope is only 454 // active for WebSpeech clients. 455 scoped_ptr<uint8[]> audio_data(new uint8[size]); 456 memcpy(audio_data.get(), data, size); 457 458 // Ownership of the audio buffer will be with the callback until it is run, 459 // when ownership is passed to the callback function. 460 task_runner_->PostTask(FROM_HERE, base::Bind( 461 &AudioInputController::DoOnData, this, base::Passed(&audio_data), size)); 462} 463 464void AudioInputController::DoOnData(scoped_ptr<uint8[]> data, uint32 size) { 465 DCHECK(task_runner_->BelongsToCurrentThread()); 466 if (handler_) 467 handler_->OnData(this, data.get(), size); 468} 469 470void AudioInputController::DoLogAudioLevel(float level_dbfs) { 471#if defined(AUDIO_POWER_MONITORING) 472 DCHECK(task_runner_->BelongsToCurrentThread()); 473 if (!handler_) 474 return; 475 476 std::string log_string = base::StringPrintf( 477 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs); 478 static const float kSilenceThresholdDBFS = -72.24719896f; 479 if (level_dbfs < kSilenceThresholdDBFS) 480 log_string += " <=> no audio input!"; 481 482 handler_->OnLog(this, log_string); 483#endif 484} 485 486void AudioInputController::OnError(AudioInputStream* stream) { 487 // Handle error on the audio-manager thread. 488 task_runner_->PostTask(FROM_HERE, base::Bind( 489 &AudioInputController::DoReportError, this)); 490} 491 492void AudioInputController::DoStopCloseAndClearStream() { 493 DCHECK(task_runner_->BelongsToCurrentThread()); 494 495 // Allow calling unconditionally and bail if we don't have a stream to close. 496 if (stream_ != NULL) { 497 stream_->Stop(); 498 stream_->Close(); 499 stream_ = NULL; 500 } 501 502 // The event handler should not be touched after the stream has been closed. 503 handler_ = NULL; 504} 505 506void AudioInputController::SetDataIsActive(bool enabled) { 507 base::subtle::Release_Store(&data_is_active_, enabled); 508} 509 510bool AudioInputController::GetDataIsActive() { 511 return (base::subtle::Acquire_Load(&data_is_active_) != false); 512} 513 514} // namespace media 515