1/*
2 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/audio_device/android/opensles_input.h"
12
13#include <assert.h>
14
15#include "webrtc/modules/audio_device/android/audio_common.h"
16#include "webrtc/modules/audio_device/android/opensles_common.h"
17#include "webrtc/modules/audio_device/android/single_rw_fifo.h"
18#include "webrtc/modules/audio_device/audio_device_buffer.h"
19#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
20#include "webrtc/system_wrappers/interface/thread_wrapper.h"
21#include "webrtc/system_wrappers/interface/trace.h"
22
23#define VOID_RETURN
24#define OPENSL_RETURN_ON_FAILURE(op, ret_val)                    \
25  do {                                                           \
26    SLresult err = (op);                                         \
27    if (err != SL_RESULT_SUCCESS) {                              \
28      WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_,          \
29                   "OpenSL error: %d", err);                     \
30      assert(false);                                             \
31      return ret_val;                                            \
32    }                                                            \
33  } while (0)
34
35static const SLEngineOption kOption[] = {
36  { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) },
37};
38
39enum {
40  kNoOverrun,
41  kOverrun,
42};
43
44namespace webrtc {
45
46OpenSlesInput::OpenSlesInput(
47    const int32_t id, PlayoutDelayProvider* delay_provider)
48    : id_(id),
49      delay_provider_(delay_provider),
50      initialized_(false),
51      mic_initialized_(false),
52      rec_initialized_(false),
53      crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
54      recording_(false),
55      num_fifo_buffers_needed_(0),
56      number_overruns_(0),
57      sles_engine_(NULL),
58      sles_engine_itf_(NULL),
59      sles_recorder_(NULL),
60      sles_recorder_itf_(NULL),
61      sles_recorder_sbq_itf_(NULL),
62      audio_buffer_(NULL),
63      active_queue_(0),
64      rec_sampling_rate_(0),
65      agc_enabled_(false),
66      recording_delay_(0) {
67}
68
69OpenSlesInput::~OpenSlesInput() {
70}
71
72int32_t OpenSlesInput::SetAndroidAudioDeviceObjects(void* javaVM,
73                                                    void* env,
74                                                    void* context) {
75  return 0;
76}
77
78void OpenSlesInput::ClearAndroidAudioDeviceObjects() {
79}
80
81int32_t OpenSlesInput::Init() {
82  assert(!initialized_);
83
84  // Set up OpenSL engine.
85  OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0,
86                                          NULL, NULL),
87                           -1);
88  OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_,
89                                                    SL_BOOLEAN_FALSE),
90                           -1);
91  OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_,
92                                                         SL_IID_ENGINE,
93                                                         &sles_engine_itf_),
94                           -1);
95
96  if (InitSampleRate() != 0) {
97    return -1;
98  }
99  AllocateBuffers();
100  initialized_ = true;
101  return 0;
102}
103
104int32_t OpenSlesInput::Terminate() {
105  // It is assumed that the caller has stopped recording before terminating.
106  assert(!recording_);
107  (*sles_engine_)->Destroy(sles_engine_);
108  initialized_ = false;
109  mic_initialized_ = false;
110  rec_initialized_ = false;
111  return 0;
112}
113
114int32_t OpenSlesInput::RecordingDeviceName(uint16_t index,
115                                           char name[kAdmMaxDeviceNameSize],
116                                           char guid[kAdmMaxGuidSize]) {
117  assert(index == 0);
118  // Empty strings.
119  name[0] = '\0';
120  guid[0] = '\0';
121  return 0;
122}
123
124int32_t OpenSlesInput::SetRecordingDevice(uint16_t index) {
125  assert(index == 0);
126  return 0;
127}
128
129int32_t OpenSlesInput::RecordingIsAvailable(bool& available) {  // NOLINT
130  available = true;
131  return 0;
132}
133
134int32_t OpenSlesInput::InitRecording() {
135  assert(initialized_);
136  rec_initialized_ = true;
137  return 0;
138}
139
140int32_t OpenSlesInput::StartRecording() {
141  assert(rec_initialized_);
142  assert(!recording_);
143  if (!CreateAudioRecorder()) {
144    return -1;
145  }
146  // Setup to receive buffer queue event callbacks.
147  OPENSL_RETURN_ON_FAILURE(
148      (*sles_recorder_sbq_itf_)->RegisterCallback(
149          sles_recorder_sbq_itf_,
150          RecorderSimpleBufferQueueCallback,
151          this),
152      -1);
153
154  if (!EnqueueAllBuffers()) {
155    return -1;
156  }
157
158  {
159    // To prevent the compiler from e.g. optimizing the code to
160    // recording_ = StartCbThreads() which wouldn't have been thread safe.
161    CriticalSectionScoped lock(crit_sect_.get());
162    recording_ = true;
163  }
164  if (!StartCbThreads()) {
165    recording_ = false;
166    return -1;
167  }
168  return 0;
169}
170
171int32_t OpenSlesInput::StopRecording() {
172  StopCbThreads();
173  DestroyAudioRecorder();
174  recording_ = false;
175  return 0;
176}
177
178int32_t OpenSlesInput::SetAGC(bool enable) {
179  agc_enabled_ = enable;
180  return 0;
181}
182
183int32_t OpenSlesInput::InitMicrophone() {
184  assert(initialized_);
185  assert(!recording_);
186  mic_initialized_ = true;
187  return 0;
188}
189
190int32_t OpenSlesInput::MicrophoneVolumeIsAvailable(bool& available) {  // NOLINT
191  available = false;
192  return 0;
193}
194
195int32_t OpenSlesInput::MinMicrophoneVolume(
196    uint32_t& minVolume) const {  // NOLINT
197  minVolume = 0;
198  return 0;
199}
200
201int32_t OpenSlesInput::MicrophoneVolumeStepSize(
202    uint16_t& stepSize) const {
203  stepSize = 1;
204  return 0;
205}
206
207int32_t OpenSlesInput::MicrophoneMuteIsAvailable(bool& available) {  // NOLINT
208  available = false;  // Mic mute not supported on Android
209  return 0;
210}
211
212int32_t OpenSlesInput::MicrophoneBoostIsAvailable(bool& available) {  // NOLINT
213  available = false;  // Mic boost not supported on Android.
214  return 0;
215}
216
217int32_t OpenSlesInput::SetMicrophoneBoost(bool enable) {
218  assert(false);
219  return -1;  // Not supported
220}
221
222int32_t OpenSlesInput::MicrophoneBoost(bool& enabled) const {  // NOLINT
223  assert(false);
224  return -1;  // Not supported
225}
226
227int32_t OpenSlesInput::StereoRecordingIsAvailable(bool& available) {  // NOLINT
228  available = false;  // Stereo recording not supported on Android.
229  return 0;
230}
231
232int32_t OpenSlesInput::StereoRecording(bool& enabled) const {  // NOLINT
233  enabled = false;
234  return 0;
235}
236
237int32_t OpenSlesInput::RecordingDelay(uint16_t& delayMS) const {  // NOLINT
238  delayMS = recording_delay_;
239  return 0;
240}
241
242void OpenSlesInput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
243  audio_buffer_ = audioBuffer;
244}
245
246int OpenSlesInput::InitSampleRate() {
247  UpdateSampleRate();
248  audio_buffer_->SetRecordingSampleRate(rec_sampling_rate_);
249  audio_buffer_->SetRecordingChannels(kNumChannels);
250  UpdateRecordingDelay();
251  return 0;
252}
253
254int OpenSlesInput::buffer_size_samples() const {
255  // Since there is no low latency recording, use buffer size corresponding to
256  // 10ms of data since that's the framesize WebRTC uses. Getting any other
257  // size would require patching together buffers somewhere before passing them
258  // to WebRTC.
259  return rec_sampling_rate_ * 10 / 1000;
260}
261
262int OpenSlesInput::buffer_size_bytes() const {
263  return buffer_size_samples() * kNumChannels * sizeof(int16_t);
264}
265
266void OpenSlesInput::UpdateRecordingDelay() {
267  // TODO(hellner): Add accurate delay estimate.
268  // On average half the current buffer will have been filled with audio.
269  int outstanding_samples =
270      (TotalBuffersUsed() - 0.5) * buffer_size_samples();
271  recording_delay_ = outstanding_samples / (rec_sampling_rate_ / 1000);
272}
273
274void OpenSlesInput::UpdateSampleRate() {
275  rec_sampling_rate_ = audio_manager_.low_latency_supported() ?
276      audio_manager_.native_output_sample_rate() : kDefaultSampleRate;
277}
278
279void OpenSlesInput::CalculateNumFifoBuffersNeeded() {
280  // Buffer size is 10ms of data.
281  num_fifo_buffers_needed_ = kNum10MsToBuffer;
282}
283
284void OpenSlesInput::AllocateBuffers() {
285  // Allocate FIFO to handle passing buffers between processing and OpenSL
286  // threads.
287  CalculateNumFifoBuffersNeeded();
288  assert(num_fifo_buffers_needed_ > 0);
289  fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_));
290
291  // Allocate the memory area to be used.
292  rec_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]);
293  for (int i = 0; i < TotalBuffersUsed(); ++i) {
294    rec_buf_[i].reset(new int8_t[buffer_size_bytes()]);
295  }
296}
297
298int OpenSlesInput::TotalBuffersUsed() const {
299  return num_fifo_buffers_needed_ + kNumOpenSlBuffers;
300}
301
302bool OpenSlesInput::EnqueueAllBuffers() {
303  active_queue_ = 0;
304  number_overruns_ = 0;
305  for (int i = 0; i < kNumOpenSlBuffers; ++i) {
306    memset(rec_buf_[i].get(), 0, buffer_size_bytes());
307    OPENSL_RETURN_ON_FAILURE(
308        (*sles_recorder_sbq_itf_)->Enqueue(
309            sles_recorder_sbq_itf_,
310            reinterpret_cast<void*>(rec_buf_[i].get()),
311            buffer_size_bytes()),
312        false);
313  }
314  // In case of underrun the fifo will be at capacity. In case of first enqueue
315  // no audio can have been returned yet meaning fifo must be empty. Any other
316  // values are unexpected.
317  assert(fifo_->size() == fifo_->capacity() ||
318         fifo_->size() == 0);
319  // OpenSL recording has been stopped. I.e. only this thread is touching
320  // |fifo_|.
321  while (fifo_->size() != 0) {
322    // Clear the fifo.
323    fifo_->Pop();
324  }
325  return true;
326}
327
328bool OpenSlesInput::CreateAudioRecorder() {
329  if (!event_.Start()) {
330    assert(false);
331    return false;
332  }
333  SLDataLocator_IODevice micLocator = {
334    SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT,
335    SL_DEFAULTDEVICEID_AUDIOINPUT, NULL };
336  SLDataSource audio_source = { &micLocator, NULL };
337
338  SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = {
339    SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
340    static_cast<SLuint32>(TotalBuffersUsed())
341  };
342  SLDataFormat_PCM configuration =
343      webrtc_opensl::CreatePcmConfiguration(rec_sampling_rate_);
344  SLDataSink audio_sink = { &simple_buf_queue, &configuration };
345
346  // Interfaces for recording android audio data and Android are needed.
347  // Note the interfaces still need to be initialized. This only tells OpenSl
348  // that the interfaces will be needed at some point.
349  const SLInterfaceID id[kNumInterfaces] = {
350    SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION };
351  const SLboolean req[kNumInterfaces] = {
352    SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
353  OPENSL_RETURN_ON_FAILURE(
354      (*sles_engine_itf_)->CreateAudioRecorder(sles_engine_itf_,
355                                               &sles_recorder_,
356                                               &audio_source,
357                                               &audio_sink,
358                                               kNumInterfaces,
359                                               id,
360                                               req),
361      false);
362
363  // Realize the recorder in synchronous mode.
364  OPENSL_RETURN_ON_FAILURE((*sles_recorder_)->Realize(sles_recorder_,
365                                                      SL_BOOLEAN_FALSE),
366                           false);
367  OPENSL_RETURN_ON_FAILURE(
368      (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_RECORD,
369                                      static_cast<void*>(&sles_recorder_itf_)),
370      false);
371  OPENSL_RETURN_ON_FAILURE(
372      (*sles_recorder_)->GetInterface(
373          sles_recorder_,
374          SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
375          static_cast<void*>(&sles_recorder_sbq_itf_)),
376      false);
377  return true;
378}
379
380void OpenSlesInput::DestroyAudioRecorder() {
381  event_.Stop();
382  if (sles_recorder_sbq_itf_) {
383    // Release all buffers currently queued up.
384    OPENSL_RETURN_ON_FAILURE(
385        (*sles_recorder_sbq_itf_)->Clear(sles_recorder_sbq_itf_),
386        VOID_RETURN);
387    sles_recorder_sbq_itf_ = NULL;
388  }
389  sles_recorder_itf_ = NULL;
390
391  if (sles_recorder_) {
392    (*sles_recorder_)->Destroy(sles_recorder_);
393    sles_recorder_ = NULL;
394  }
395}
396
397bool OpenSlesInput::HandleOverrun(int event_id, int event_msg) {
398  if (!recording_) {
399    return false;
400  }
401  if (event_id == kNoOverrun) {
402    return false;
403  }
404  WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio overrun");
405  assert(event_id == kOverrun);
406  assert(event_msg > 0);
407  // Wait for all enqueued buffers be flushed.
408  if (event_msg != kNumOpenSlBuffers) {
409    return true;
410  }
411  // All buffers passed to OpenSL have been flushed. Restart the audio from
412  // scratch.
413  // No need to check sles_recorder_itf_ as recording_ would be false before it
414  // is set to NULL.
415  OPENSL_RETURN_ON_FAILURE(
416      (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
417                                            SL_RECORDSTATE_STOPPED),
418      true);
419  EnqueueAllBuffers();
420  OPENSL_RETURN_ON_FAILURE(
421      (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
422                                            SL_RECORDSTATE_RECORDING),
423      true);
424  return true;
425}
426
427void OpenSlesInput::RecorderSimpleBufferQueueCallback(
428    SLAndroidSimpleBufferQueueItf queue_itf,
429    void* context) {
430  OpenSlesInput* audio_device = reinterpret_cast<OpenSlesInput*>(context);
431  audio_device->RecorderSimpleBufferQueueCallbackHandler(queue_itf);
432}
433
434void OpenSlesInput::RecorderSimpleBufferQueueCallbackHandler(
435    SLAndroidSimpleBufferQueueItf queue_itf) {
436  if (fifo_->size() >= fifo_->capacity() || number_overruns_ > 0) {
437    ++number_overruns_;
438    event_.SignalEvent(kOverrun, number_overruns_);
439    return;
440  }
441  int8_t* audio = rec_buf_[active_queue_].get();
442  // There is at least one spot available in the fifo.
443  fifo_->Push(audio);
444  active_queue_ = (active_queue_ + 1) % TotalBuffersUsed();
445  event_.SignalEvent(kNoOverrun, 0);
446  // active_queue_ is indexing the next buffer to record to. Since the current
447  // buffer has been recorded it means that the buffer index
448  // kNumOpenSlBuffers - 1 past |active_queue_| contains the next free buffer.
449  // Since |fifo_| wasn't at capacity, at least one buffer is free to be used.
450  int next_free_buffer =
451      (active_queue_ + kNumOpenSlBuffers - 1) % TotalBuffersUsed();
452  OPENSL_RETURN_ON_FAILURE(
453      (*sles_recorder_sbq_itf_)->Enqueue(
454          sles_recorder_sbq_itf_,
455          reinterpret_cast<void*>(rec_buf_[next_free_buffer].get()),
456          buffer_size_bytes()),
457      VOID_RETURN);
458}
459
460bool OpenSlesInput::StartCbThreads() {
461  rec_thread_.reset(ThreadWrapper::CreateThread(CbThread,
462                                                this,
463                                                kRealtimePriority,
464                                                "opensl_rec_thread"));
465  assert(rec_thread_.get());
466  unsigned int thread_id = 0;
467  if (!rec_thread_->Start(thread_id)) {
468    assert(false);
469    return false;
470  }
471  OPENSL_RETURN_ON_FAILURE(
472      (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
473                                            SL_RECORDSTATE_RECORDING),
474      false);
475  return true;
476}
477
478void OpenSlesInput::StopCbThreads() {
479  {
480    CriticalSectionScoped lock(crit_sect_.get());
481    recording_ = false;
482  }
483  if (sles_recorder_itf_) {
484    OPENSL_RETURN_ON_FAILURE(
485        (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
486                                              SL_RECORDSTATE_STOPPED),
487        VOID_RETURN);
488  }
489  if (rec_thread_.get() == NULL) {
490    return;
491  }
492  event_.Stop();
493  if (rec_thread_->Stop()) {
494    rec_thread_.reset();
495  } else {
496    assert(false);
497  }
498}
499
500bool OpenSlesInput::CbThread(void* context) {
501  return reinterpret_cast<OpenSlesInput*>(context)->CbThreadImpl();
502}
503
504bool OpenSlesInput::CbThreadImpl() {
505  int event_id;
506  int event_msg;
507  // event_ must not be waited on while a lock has been taken.
508  event_.WaitOnEvent(&event_id, &event_msg);
509
510  CriticalSectionScoped lock(crit_sect_.get());
511  if (HandleOverrun(event_id, event_msg)) {
512    return recording_;
513  }
514  // If the fifo_ has audio data process it.
515  while (fifo_->size() > 0 && recording_) {
516    int8_t* audio = fifo_->Pop();
517    audio_buffer_->SetRecordedBuffer(audio, buffer_size_samples());
518    audio_buffer_->SetVQEData(delay_provider_->PlayoutDelayMs(),
519                              recording_delay_, 0);
520    audio_buffer_->DeliverRecordedData();
521  }
522  return recording_;
523}
524
525}  // namespace webrtc
526