1/* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#include "webrtc/modules/audio_device/android/opensles_input.h" 12 13#include <assert.h> 14 15#include "webrtc/modules/audio_device/android/audio_common.h" 16#include "webrtc/modules/audio_device/android/opensles_common.h" 17#include "webrtc/modules/audio_device/android/single_rw_fifo.h" 18#include "webrtc/modules/audio_device/audio_device_buffer.h" 19#include "webrtc/system_wrappers/interface/critical_section_wrapper.h" 20#include "webrtc/system_wrappers/interface/thread_wrapper.h" 21#include "webrtc/system_wrappers/interface/trace.h" 22 23#define VOID_RETURN 24#define OPENSL_RETURN_ON_FAILURE(op, ret_val) \ 25 do { \ 26 SLresult err = (op); \ 27 if (err != SL_RESULT_SUCCESS) { \ 28 WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, \ 29 "OpenSL error: %d", err); \ 30 assert(false); \ 31 return ret_val; \ 32 } \ 33 } while (0) 34 35static const SLEngineOption kOption[] = { 36 { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) }, 37}; 38 39enum { 40 kNoOverrun, 41 kOverrun, 42}; 43 44namespace webrtc { 45 46OpenSlesInput::OpenSlesInput( 47 const int32_t id, PlayoutDelayProvider* delay_provider) 48 : id_(id), 49 delay_provider_(delay_provider), 50 initialized_(false), 51 mic_initialized_(false), 52 rec_initialized_(false), 53 crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), 54 recording_(false), 55 num_fifo_buffers_needed_(0), 56 number_overruns_(0), 57 sles_engine_(NULL), 58 sles_engine_itf_(NULL), 59 sles_recorder_(NULL), 60 sles_recorder_itf_(NULL), 61 sles_recorder_sbq_itf_(NULL), 62 audio_buffer_(NULL), 63 active_queue_(0), 64 rec_sampling_rate_(0), 65 agc_enabled_(false), 66 recording_delay_(0) { 67} 68 69OpenSlesInput::~OpenSlesInput() { 70} 71 72int32_t OpenSlesInput::SetAndroidAudioDeviceObjects(void* javaVM, 73 void* env, 74 void* context) { 75 return 0; 76} 77 78void OpenSlesInput::ClearAndroidAudioDeviceObjects() { 79} 80 81int32_t OpenSlesInput::Init() { 82 assert(!initialized_); 83 84 // Set up OpenSL engine. 85 OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0, 86 NULL, NULL), 87 -1); 88 OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_, 89 SL_BOOLEAN_FALSE), 90 -1); 91 OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_, 92 SL_IID_ENGINE, 93 &sles_engine_itf_), 94 -1); 95 96 if (InitSampleRate() != 0) { 97 return -1; 98 } 99 AllocateBuffers(); 100 initialized_ = true; 101 return 0; 102} 103 104int32_t OpenSlesInput::Terminate() { 105 // It is assumed that the caller has stopped recording before terminating. 106 assert(!recording_); 107 (*sles_engine_)->Destroy(sles_engine_); 108 initialized_ = false; 109 mic_initialized_ = false; 110 rec_initialized_ = false; 111 return 0; 112} 113 114int32_t OpenSlesInput::RecordingDeviceName(uint16_t index, 115 char name[kAdmMaxDeviceNameSize], 116 char guid[kAdmMaxGuidSize]) { 117 assert(index == 0); 118 // Empty strings. 119 name[0] = '\0'; 120 guid[0] = '\0'; 121 return 0; 122} 123 124int32_t OpenSlesInput::SetRecordingDevice(uint16_t index) { 125 assert(index == 0); 126 return 0; 127} 128 129int32_t OpenSlesInput::RecordingIsAvailable(bool& available) { // NOLINT 130 available = true; 131 return 0; 132} 133 134int32_t OpenSlesInput::InitRecording() { 135 assert(initialized_); 136 rec_initialized_ = true; 137 return 0; 138} 139 140int32_t OpenSlesInput::StartRecording() { 141 assert(rec_initialized_); 142 assert(!recording_); 143 if (!CreateAudioRecorder()) { 144 return -1; 145 } 146 // Setup to receive buffer queue event callbacks. 147 OPENSL_RETURN_ON_FAILURE( 148 (*sles_recorder_sbq_itf_)->RegisterCallback( 149 sles_recorder_sbq_itf_, 150 RecorderSimpleBufferQueueCallback, 151 this), 152 -1); 153 154 if (!EnqueueAllBuffers()) { 155 return -1; 156 } 157 158 { 159 // To prevent the compiler from e.g. optimizing the code to 160 // recording_ = StartCbThreads() which wouldn't have been thread safe. 161 CriticalSectionScoped lock(crit_sect_.get()); 162 recording_ = true; 163 } 164 if (!StartCbThreads()) { 165 recording_ = false; 166 return -1; 167 } 168 return 0; 169} 170 171int32_t OpenSlesInput::StopRecording() { 172 StopCbThreads(); 173 DestroyAudioRecorder(); 174 recording_ = false; 175 return 0; 176} 177 178int32_t OpenSlesInput::SetAGC(bool enable) { 179 agc_enabled_ = enable; 180 return 0; 181} 182 183int32_t OpenSlesInput::InitMicrophone() { 184 assert(initialized_); 185 assert(!recording_); 186 mic_initialized_ = true; 187 return 0; 188} 189 190int32_t OpenSlesInput::MicrophoneVolumeIsAvailable(bool& available) { // NOLINT 191 available = false; 192 return 0; 193} 194 195int32_t OpenSlesInput::MinMicrophoneVolume( 196 uint32_t& minVolume) const { // NOLINT 197 minVolume = 0; 198 return 0; 199} 200 201int32_t OpenSlesInput::MicrophoneVolumeStepSize( 202 uint16_t& stepSize) const { 203 stepSize = 1; 204 return 0; 205} 206 207int32_t OpenSlesInput::MicrophoneMuteIsAvailable(bool& available) { // NOLINT 208 available = false; // Mic mute not supported on Android 209 return 0; 210} 211 212int32_t OpenSlesInput::MicrophoneBoostIsAvailable(bool& available) { // NOLINT 213 available = false; // Mic boost not supported on Android. 214 return 0; 215} 216 217int32_t OpenSlesInput::SetMicrophoneBoost(bool enable) { 218 assert(false); 219 return -1; // Not supported 220} 221 222int32_t OpenSlesInput::MicrophoneBoost(bool& enabled) const { // NOLINT 223 assert(false); 224 return -1; // Not supported 225} 226 227int32_t OpenSlesInput::StereoRecordingIsAvailable(bool& available) { // NOLINT 228 available = false; // Stereo recording not supported on Android. 229 return 0; 230} 231 232int32_t OpenSlesInput::StereoRecording(bool& enabled) const { // NOLINT 233 enabled = false; 234 return 0; 235} 236 237int32_t OpenSlesInput::RecordingDelay(uint16_t& delayMS) const { // NOLINT 238 delayMS = recording_delay_; 239 return 0; 240} 241 242void OpenSlesInput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { 243 audio_buffer_ = audioBuffer; 244} 245 246int OpenSlesInput::InitSampleRate() { 247 UpdateSampleRate(); 248 audio_buffer_->SetRecordingSampleRate(rec_sampling_rate_); 249 audio_buffer_->SetRecordingChannels(kNumChannels); 250 UpdateRecordingDelay(); 251 return 0; 252} 253 254int OpenSlesInput::buffer_size_samples() const { 255 // Since there is no low latency recording, use buffer size corresponding to 256 // 10ms of data since that's the framesize WebRTC uses. Getting any other 257 // size would require patching together buffers somewhere before passing them 258 // to WebRTC. 259 return rec_sampling_rate_ * 10 / 1000; 260} 261 262int OpenSlesInput::buffer_size_bytes() const { 263 return buffer_size_samples() * kNumChannels * sizeof(int16_t); 264} 265 266void OpenSlesInput::UpdateRecordingDelay() { 267 // TODO(hellner): Add accurate delay estimate. 268 // On average half the current buffer will have been filled with audio. 269 int outstanding_samples = 270 (TotalBuffersUsed() - 0.5) * buffer_size_samples(); 271 recording_delay_ = outstanding_samples / (rec_sampling_rate_ / 1000); 272} 273 274void OpenSlesInput::UpdateSampleRate() { 275 rec_sampling_rate_ = audio_manager_.low_latency_supported() ? 276 audio_manager_.native_output_sample_rate() : kDefaultSampleRate; 277} 278 279void OpenSlesInput::CalculateNumFifoBuffersNeeded() { 280 // Buffer size is 10ms of data. 281 num_fifo_buffers_needed_ = kNum10MsToBuffer; 282} 283 284void OpenSlesInput::AllocateBuffers() { 285 // Allocate FIFO to handle passing buffers between processing and OpenSL 286 // threads. 287 CalculateNumFifoBuffersNeeded(); 288 assert(num_fifo_buffers_needed_ > 0); 289 fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_)); 290 291 // Allocate the memory area to be used. 292 rec_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]); 293 for (int i = 0; i < TotalBuffersUsed(); ++i) { 294 rec_buf_[i].reset(new int8_t[buffer_size_bytes()]); 295 } 296} 297 298int OpenSlesInput::TotalBuffersUsed() const { 299 return num_fifo_buffers_needed_ + kNumOpenSlBuffers; 300} 301 302bool OpenSlesInput::EnqueueAllBuffers() { 303 active_queue_ = 0; 304 number_overruns_ = 0; 305 for (int i = 0; i < kNumOpenSlBuffers; ++i) { 306 memset(rec_buf_[i].get(), 0, buffer_size_bytes()); 307 OPENSL_RETURN_ON_FAILURE( 308 (*sles_recorder_sbq_itf_)->Enqueue( 309 sles_recorder_sbq_itf_, 310 reinterpret_cast<void*>(rec_buf_[i].get()), 311 buffer_size_bytes()), 312 false); 313 } 314 // In case of underrun the fifo will be at capacity. In case of first enqueue 315 // no audio can have been returned yet meaning fifo must be empty. Any other 316 // values are unexpected. 317 assert(fifo_->size() == fifo_->capacity() || 318 fifo_->size() == 0); 319 // OpenSL recording has been stopped. I.e. only this thread is touching 320 // |fifo_|. 321 while (fifo_->size() != 0) { 322 // Clear the fifo. 323 fifo_->Pop(); 324 } 325 return true; 326} 327 328bool OpenSlesInput::CreateAudioRecorder() { 329 if (!event_.Start()) { 330 assert(false); 331 return false; 332 } 333 SLDataLocator_IODevice micLocator = { 334 SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT, 335 SL_DEFAULTDEVICEID_AUDIOINPUT, NULL }; 336 SLDataSource audio_source = { &micLocator, NULL }; 337 338 SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = { 339 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 340 static_cast<SLuint32>(TotalBuffersUsed()) 341 }; 342 SLDataFormat_PCM configuration = 343 webrtc_opensl::CreatePcmConfiguration(rec_sampling_rate_); 344 SLDataSink audio_sink = { &simple_buf_queue, &configuration }; 345 346 // Interfaces for recording android audio data and Android are needed. 347 // Note the interfaces still need to be initialized. This only tells OpenSl 348 // that the interfaces will be needed at some point. 349 const SLInterfaceID id[kNumInterfaces] = { 350 SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION }; 351 const SLboolean req[kNumInterfaces] = { 352 SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE }; 353 OPENSL_RETURN_ON_FAILURE( 354 (*sles_engine_itf_)->CreateAudioRecorder(sles_engine_itf_, 355 &sles_recorder_, 356 &audio_source, 357 &audio_sink, 358 kNumInterfaces, 359 id, 360 req), 361 false); 362 363 // Realize the recorder in synchronous mode. 364 OPENSL_RETURN_ON_FAILURE((*sles_recorder_)->Realize(sles_recorder_, 365 SL_BOOLEAN_FALSE), 366 false); 367 OPENSL_RETURN_ON_FAILURE( 368 (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_RECORD, 369 static_cast<void*>(&sles_recorder_itf_)), 370 false); 371 OPENSL_RETURN_ON_FAILURE( 372 (*sles_recorder_)->GetInterface( 373 sles_recorder_, 374 SL_IID_ANDROIDSIMPLEBUFFERQUEUE, 375 static_cast<void*>(&sles_recorder_sbq_itf_)), 376 false); 377 return true; 378} 379 380void OpenSlesInput::DestroyAudioRecorder() { 381 event_.Stop(); 382 if (sles_recorder_sbq_itf_) { 383 // Release all buffers currently queued up. 384 OPENSL_RETURN_ON_FAILURE( 385 (*sles_recorder_sbq_itf_)->Clear(sles_recorder_sbq_itf_), 386 VOID_RETURN); 387 sles_recorder_sbq_itf_ = NULL; 388 } 389 sles_recorder_itf_ = NULL; 390 391 if (sles_recorder_) { 392 (*sles_recorder_)->Destroy(sles_recorder_); 393 sles_recorder_ = NULL; 394 } 395} 396 397bool OpenSlesInput::HandleOverrun(int event_id, int event_msg) { 398 if (!recording_) { 399 return false; 400 } 401 if (event_id == kNoOverrun) { 402 return false; 403 } 404 WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio overrun"); 405 assert(event_id == kOverrun); 406 assert(event_msg > 0); 407 // Wait for all enqueued buffers be flushed. 408 if (event_msg != kNumOpenSlBuffers) { 409 return true; 410 } 411 // All buffers passed to OpenSL have been flushed. Restart the audio from 412 // scratch. 413 // No need to check sles_recorder_itf_ as recording_ would be false before it 414 // is set to NULL. 415 OPENSL_RETURN_ON_FAILURE( 416 (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_, 417 SL_RECORDSTATE_STOPPED), 418 true); 419 EnqueueAllBuffers(); 420 OPENSL_RETURN_ON_FAILURE( 421 (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_, 422 SL_RECORDSTATE_RECORDING), 423 true); 424 return true; 425} 426 427void OpenSlesInput::RecorderSimpleBufferQueueCallback( 428 SLAndroidSimpleBufferQueueItf queue_itf, 429 void* context) { 430 OpenSlesInput* audio_device = reinterpret_cast<OpenSlesInput*>(context); 431 audio_device->RecorderSimpleBufferQueueCallbackHandler(queue_itf); 432} 433 434void OpenSlesInput::RecorderSimpleBufferQueueCallbackHandler( 435 SLAndroidSimpleBufferQueueItf queue_itf) { 436 if (fifo_->size() >= fifo_->capacity() || number_overruns_ > 0) { 437 ++number_overruns_; 438 event_.SignalEvent(kOverrun, number_overruns_); 439 return; 440 } 441 int8_t* audio = rec_buf_[active_queue_].get(); 442 // There is at least one spot available in the fifo. 443 fifo_->Push(audio); 444 active_queue_ = (active_queue_ + 1) % TotalBuffersUsed(); 445 event_.SignalEvent(kNoOverrun, 0); 446 // active_queue_ is indexing the next buffer to record to. Since the current 447 // buffer has been recorded it means that the buffer index 448 // kNumOpenSlBuffers - 1 past |active_queue_| contains the next free buffer. 449 // Since |fifo_| wasn't at capacity, at least one buffer is free to be used. 450 int next_free_buffer = 451 (active_queue_ + kNumOpenSlBuffers - 1) % TotalBuffersUsed(); 452 OPENSL_RETURN_ON_FAILURE( 453 (*sles_recorder_sbq_itf_)->Enqueue( 454 sles_recorder_sbq_itf_, 455 reinterpret_cast<void*>(rec_buf_[next_free_buffer].get()), 456 buffer_size_bytes()), 457 VOID_RETURN); 458} 459 460bool OpenSlesInput::StartCbThreads() { 461 rec_thread_.reset(ThreadWrapper::CreateThread(CbThread, 462 this, 463 kRealtimePriority, 464 "opensl_rec_thread")); 465 assert(rec_thread_.get()); 466 unsigned int thread_id = 0; 467 if (!rec_thread_->Start(thread_id)) { 468 assert(false); 469 return false; 470 } 471 OPENSL_RETURN_ON_FAILURE( 472 (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_, 473 SL_RECORDSTATE_RECORDING), 474 false); 475 return true; 476} 477 478void OpenSlesInput::StopCbThreads() { 479 { 480 CriticalSectionScoped lock(crit_sect_.get()); 481 recording_ = false; 482 } 483 if (sles_recorder_itf_) { 484 OPENSL_RETURN_ON_FAILURE( 485 (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_, 486 SL_RECORDSTATE_STOPPED), 487 VOID_RETURN); 488 } 489 if (rec_thread_.get() == NULL) { 490 return; 491 } 492 event_.Stop(); 493 if (rec_thread_->Stop()) { 494 rec_thread_.reset(); 495 } else { 496 assert(false); 497 } 498} 499 500bool OpenSlesInput::CbThread(void* context) { 501 return reinterpret_cast<OpenSlesInput*>(context)->CbThreadImpl(); 502} 503 504bool OpenSlesInput::CbThreadImpl() { 505 int event_id; 506 int event_msg; 507 // event_ must not be waited on while a lock has been taken. 508 event_.WaitOnEvent(&event_id, &event_msg); 509 510 CriticalSectionScoped lock(crit_sect_.get()); 511 if (HandleOverrun(event_id, event_msg)) { 512 return recording_; 513 } 514 // If the fifo_ has audio data process it. 515 while (fifo_->size() > 0 && recording_) { 516 int8_t* audio = fifo_->Pop(); 517 audio_buffer_->SetRecordedBuffer(audio, buffer_size_samples()); 518 audio_buffer_->SetVQEData(delay_provider_->PlayoutDelayMs(), 519 recording_delay_, 0); 520 audio_buffer_->DeliverRecordedData(); 521 } 522 return recording_; 523} 524 525} // namespace webrtc 526