1/*
2 *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/audio_device/android/opensles_player.h"
12
13#include <android/log.h>
14
15#include "webrtc/base/arraysize.h"
16#include "webrtc/base/checks.h"
17#include "webrtc/base/format_macros.h"
18#include "webrtc/base/timeutils.h"
19#include "webrtc/modules/audio_device/android/audio_manager.h"
20#include "webrtc/modules/audio_device/fine_audio_buffer.h"
21
22#define TAG "OpenSLESPlayer"
23#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
24#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
25#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
26#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
27#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
28
29#define RETURN_ON_ERROR(op, ...)        \
30  do {                                  \
31    SLresult err = (op);                \
32    if (err != SL_RESULT_SUCCESS) {     \
33      ALOGE("%s failed: %d", #op, err); \
34      return __VA_ARGS__;               \
35    }                                   \
36  } while (0)
37
38namespace webrtc {
39
40OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager)
41    : audio_parameters_(audio_manager->GetPlayoutAudioParameters()),
42      audio_device_buffer_(NULL),
43      initialized_(false),
44      playing_(false),
45      bytes_per_buffer_(0),
46      buffer_index_(0),
47      engine_(nullptr),
48      player_(nullptr),
49      simple_buffer_queue_(nullptr),
50      volume_(nullptr),
51      last_play_time_(0) {
52  ALOGD("ctor%s", GetThreadInfo().c_str());
53  // Use native audio output parameters provided by the audio manager and
54  // define the PCM format structure.
55  pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
56                                       audio_parameters_.sample_rate(),
57                                       audio_parameters_.bits_per_sample());
58  // Detach from this thread since we want to use the checker to verify calls
59  // from the internal  audio thread.
60  thread_checker_opensles_.DetachFromThread();
61}
62
63OpenSLESPlayer::~OpenSLESPlayer() {
64  ALOGD("dtor%s", GetThreadInfo().c_str());
65  RTC_DCHECK(thread_checker_.CalledOnValidThread());
66  Terminate();
67  DestroyAudioPlayer();
68  DestroyMix();
69  DestroyEngine();
70  RTC_DCHECK(!engine_object_.Get());
71  RTC_DCHECK(!engine_);
72  RTC_DCHECK(!output_mix_.Get());
73  RTC_DCHECK(!player_);
74  RTC_DCHECK(!simple_buffer_queue_);
75  RTC_DCHECK(!volume_);
76}
77
78int OpenSLESPlayer::Init() {
79  ALOGD("Init%s", GetThreadInfo().c_str());
80  RTC_DCHECK(thread_checker_.CalledOnValidThread());
81  return 0;
82}
83
84int OpenSLESPlayer::Terminate() {
85  ALOGD("Terminate%s", GetThreadInfo().c_str());
86  RTC_DCHECK(thread_checker_.CalledOnValidThread());
87  StopPlayout();
88  return 0;
89}
90
91int OpenSLESPlayer::InitPlayout() {
92  ALOGD("InitPlayout%s", GetThreadInfo().c_str());
93  RTC_DCHECK(thread_checker_.CalledOnValidThread());
94  RTC_DCHECK(!initialized_);
95  RTC_DCHECK(!playing_);
96  CreateEngine();
97  CreateMix();
98  initialized_ = true;
99  buffer_index_ = 0;
100  last_play_time_ = rtc::Time();
101  return 0;
102}
103
104int OpenSLESPlayer::StartPlayout() {
105  ALOGD("StartPlayout%s", GetThreadInfo().c_str());
106  RTC_DCHECK(thread_checker_.CalledOnValidThread());
107  RTC_DCHECK(initialized_);
108  RTC_DCHECK(!playing_);
109  // The number of lower latency audio players is limited, hence we create the
110  // audio player in Start() and destroy it in Stop().
111  CreateAudioPlayer();
112  // Fill up audio buffers to avoid initial glitch and to ensure that playback
113  // starts when mode is later changed to SL_PLAYSTATE_PLAYING.
114  // TODO(henrika): we can save some delay by only making one call to
115  // EnqueuePlayoutData. Most likely not worth the risk of adding a glitch.
116  for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
117    EnqueuePlayoutData();
118  }
119  // Start streaming data by setting the play state to SL_PLAYSTATE_PLAYING.
120  // For a player object, when the object is in the SL_PLAYSTATE_PLAYING
121  // state, adding buffers will implicitly start playback.
122  RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_PLAYING), -1);
123  playing_ = (GetPlayState() == SL_PLAYSTATE_PLAYING);
124  RTC_DCHECK(playing_);
125  return 0;
126}
127
128int OpenSLESPlayer::StopPlayout() {
129  ALOGD("StopPlayout%s", GetThreadInfo().c_str());
130  RTC_DCHECK(thread_checker_.CalledOnValidThread());
131  if (!initialized_ || !playing_) {
132    return 0;
133  }
134  // Stop playing by setting the play state to SL_PLAYSTATE_STOPPED.
135  RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_STOPPED), -1);
136  // Clear the buffer queue to flush out any remaining data.
137  RETURN_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_), -1);
138#ifndef NDEBUG
139  // Verify that the buffer queue is in fact cleared as it should.
140  SLAndroidSimpleBufferQueueState buffer_queue_state;
141  (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &buffer_queue_state);
142  RTC_DCHECK_EQ(0u, buffer_queue_state.count);
143  RTC_DCHECK_EQ(0u, buffer_queue_state.index);
144#endif
145  // The number of lower latency audio players is limited, hence we create the
146  // audio player in Start() and destroy it in Stop().
147  DestroyAudioPlayer();
148  thread_checker_opensles_.DetachFromThread();
149  initialized_ = false;
150  playing_ = false;
151  return 0;
152}
153
154int OpenSLESPlayer::SpeakerVolumeIsAvailable(bool& available) {
155  available = false;
156  return 0;
157}
158
159int OpenSLESPlayer::MaxSpeakerVolume(uint32_t& maxVolume) const {
160  return -1;
161}
162
163int OpenSLESPlayer::MinSpeakerVolume(uint32_t& minVolume) const {
164  return -1;
165}
166
167int OpenSLESPlayer::SetSpeakerVolume(uint32_t volume) {
168  return -1;
169}
170
171int OpenSLESPlayer::SpeakerVolume(uint32_t& volume) const {
172  return -1;
173}
174
175void OpenSLESPlayer::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
176  ALOGD("AttachAudioBuffer");
177  RTC_DCHECK(thread_checker_.CalledOnValidThread());
178  audio_device_buffer_ = audioBuffer;
179  const int sample_rate_hz = audio_parameters_.sample_rate();
180  ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz);
181  audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz);
182  const size_t channels = audio_parameters_.channels();
183  ALOGD("SetPlayoutChannels(%" PRIuS ")", channels);
184  audio_device_buffer_->SetPlayoutChannels(channels);
185  RTC_CHECK(audio_device_buffer_);
186  AllocateDataBuffers();
187}
188
189SLDataFormat_PCM OpenSLESPlayer::CreatePCMConfiguration(
190    size_t channels,
191    int sample_rate,
192    size_t bits_per_sample) {
193  ALOGD("CreatePCMConfiguration");
194  RTC_CHECK_EQ(bits_per_sample, SL_PCMSAMPLEFORMAT_FIXED_16);
195  SLDataFormat_PCM format;
196  format.formatType = SL_DATAFORMAT_PCM;
197  format.numChannels = static_cast<SLuint32>(channels);
198  // Note that, the unit of sample rate is actually in milliHertz and not Hertz.
199  switch (sample_rate) {
200    case 8000:
201      format.samplesPerSec = SL_SAMPLINGRATE_8;
202      break;
203    case 16000:
204      format.samplesPerSec = SL_SAMPLINGRATE_16;
205      break;
206    case 22050:
207      format.samplesPerSec = SL_SAMPLINGRATE_22_05;
208      break;
209    case 32000:
210      format.samplesPerSec = SL_SAMPLINGRATE_32;
211      break;
212    case 44100:
213      format.samplesPerSec = SL_SAMPLINGRATE_44_1;
214      break;
215    case 48000:
216      format.samplesPerSec = SL_SAMPLINGRATE_48;
217      break;
218    default:
219      RTC_CHECK(false) << "Unsupported sample rate: " << sample_rate;
220  }
221  format.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16;
222  format.containerSize = SL_PCMSAMPLEFORMAT_FIXED_16;
223  format.endianness = SL_BYTEORDER_LITTLEENDIAN;
224  if (format.numChannels == 1)
225    format.channelMask = SL_SPEAKER_FRONT_CENTER;
226  else if (format.numChannels == 2)
227    format.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
228  else
229    RTC_CHECK(false) << "Unsupported number of channels: "
230                     << format.numChannels;
231  return format;
232}
233
234void OpenSLESPlayer::AllocateDataBuffers() {
235  ALOGD("AllocateDataBuffers");
236  RTC_DCHECK(thread_checker_.CalledOnValidThread());
237  RTC_DCHECK(!simple_buffer_queue_);
238  RTC_CHECK(audio_device_buffer_);
239  // Don't use the lowest possible size as native buffer size. Instead,
240  // use 10ms to better match the frame size that WebRTC uses. It will result
241  // in a reduced risk for audio glitches and also in a more "clean" sequence
242  // of callbacks from the OpenSL ES thread in to WebRTC when asking for audio
243  // to render.
244  ALOGD("lowest possible buffer size: %" PRIuS,
245      audio_parameters_.GetBytesPerBuffer());
246  bytes_per_buffer_ = audio_parameters_.GetBytesPerFrame() *
247      audio_parameters_.frames_per_10ms_buffer();
248  RTC_DCHECK_GE(bytes_per_buffer_, audio_parameters_.GetBytesPerBuffer());
249  ALOGD("native buffer size: %" PRIuS, bytes_per_buffer_);
250  // Create a modified audio buffer class which allows us to ask for any number
251  // of samples (and not only multiple of 10ms) to match the native OpenSL ES
252  // buffer size.
253  fine_buffer_.reset(new FineAudioBuffer(audio_device_buffer_,
254                                         bytes_per_buffer_,
255                                         audio_parameters_.sample_rate()));
256  // Each buffer must be of this size to avoid unnecessary memcpy while caching
257  // data between successive callbacks.
258  const size_t required_buffer_size =
259      fine_buffer_->RequiredPlayoutBufferSizeBytes();
260  ALOGD("required buffer size: %" PRIuS, required_buffer_size);
261  for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
262    audio_buffers_[i].reset(new SLint8[required_buffer_size]);
263  }
264}
265
266bool OpenSLESPlayer::CreateEngine() {
267  ALOGD("CreateEngine");
268  RTC_DCHECK(thread_checker_.CalledOnValidThread());
269  if (engine_object_.Get())
270    return true;
271  RTC_DCHECK(!engine_);
272  const SLEngineOption option[] = {
273    {SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE)}};
274  RETURN_ON_ERROR(
275      slCreateEngine(engine_object_.Receive(), 1, option, 0, NULL, NULL),
276      false);
277  RETURN_ON_ERROR(
278      engine_object_->Realize(engine_object_.Get(), SL_BOOLEAN_FALSE), false);
279  RETURN_ON_ERROR(engine_object_->GetInterface(engine_object_.Get(),
280                                               SL_IID_ENGINE, &engine_),
281                  false);
282  return true;
283}
284
285void OpenSLESPlayer::DestroyEngine() {
286  ALOGD("DestroyEngine");
287  RTC_DCHECK(thread_checker_.CalledOnValidThread());
288  if (!engine_object_.Get())
289    return;
290  engine_ = nullptr;
291  engine_object_.Reset();
292}
293
294bool OpenSLESPlayer::CreateMix() {
295  ALOGD("CreateMix");
296  RTC_DCHECK(thread_checker_.CalledOnValidThread());
297  RTC_DCHECK(engine_);
298  if (output_mix_.Get())
299    return true;
300
301  // Create the ouput mix on the engine object. No interfaces will be used.
302  RETURN_ON_ERROR((*engine_)->CreateOutputMix(engine_, output_mix_.Receive(), 0,
303                                              NULL, NULL),
304                  false);
305  RETURN_ON_ERROR(output_mix_->Realize(output_mix_.Get(), SL_BOOLEAN_FALSE),
306                  false);
307  return true;
308}
309
310void OpenSLESPlayer::DestroyMix() {
311  ALOGD("DestroyMix");
312  RTC_DCHECK(thread_checker_.CalledOnValidThread());
313  if (!output_mix_.Get())
314    return;
315  output_mix_.Reset();
316}
317
318bool OpenSLESPlayer::CreateAudioPlayer() {
319  ALOGD("CreateAudioPlayer");
320  RTC_DCHECK(thread_checker_.CalledOnValidThread());
321  RTC_DCHECK(engine_object_.Get());
322  RTC_DCHECK(output_mix_.Get());
323  if (player_object_.Get())
324    return true;
325  RTC_DCHECK(!player_);
326  RTC_DCHECK(!simple_buffer_queue_);
327  RTC_DCHECK(!volume_);
328
329  // source: Android Simple Buffer Queue Data Locator is source.
330  SLDataLocator_AndroidSimpleBufferQueue simple_buffer_queue = {
331      SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
332      static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
333  SLDataSource audio_source = {&simple_buffer_queue, &pcm_format_};
334
335  // sink: OutputMix-based data is sink.
336  SLDataLocator_OutputMix locator_output_mix = {SL_DATALOCATOR_OUTPUTMIX,
337                                                output_mix_.Get()};
338  SLDataSink audio_sink = {&locator_output_mix, NULL};
339
340  // Define interfaces that we indend to use and realize.
341  const SLInterfaceID interface_ids[] = {
342      SL_IID_ANDROIDCONFIGURATION, SL_IID_BUFFERQUEUE, SL_IID_VOLUME};
343  const SLboolean interface_required[] = {
344      SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
345
346  // Create the audio player on the engine interface.
347  RETURN_ON_ERROR(
348      (*engine_)->CreateAudioPlayer(
349          engine_, player_object_.Receive(), &audio_source, &audio_sink,
350          arraysize(interface_ids), interface_ids, interface_required),
351      false);
352
353  // Use the Android configuration interface to set platform-specific
354  // parameters. Should be done before player is realized.
355  SLAndroidConfigurationItf player_config;
356  RETURN_ON_ERROR(
357      player_object_->GetInterface(player_object_.Get(),
358                                   SL_IID_ANDROIDCONFIGURATION, &player_config),
359      false);
360  // Set audio player configuration to SL_ANDROID_STREAM_VOICE which
361  // corresponds to android.media.AudioManager.STREAM_VOICE_CALL.
362  SLint32 stream_type = SL_ANDROID_STREAM_VOICE;
363  RETURN_ON_ERROR(
364      (*player_config)
365          ->SetConfiguration(player_config, SL_ANDROID_KEY_STREAM_TYPE,
366                             &stream_type, sizeof(SLint32)),
367      false);
368
369  // Realize the audio player object after configuration has been set.
370  RETURN_ON_ERROR(
371      player_object_->Realize(player_object_.Get(), SL_BOOLEAN_FALSE), false);
372
373  // Get the SLPlayItf interface on the audio player.
374  RETURN_ON_ERROR(
375      player_object_->GetInterface(player_object_.Get(), SL_IID_PLAY, &player_),
376      false);
377
378  // Get the SLAndroidSimpleBufferQueueItf interface on the audio player.
379  RETURN_ON_ERROR(
380      player_object_->GetInterface(player_object_.Get(), SL_IID_BUFFERQUEUE,
381                                   &simple_buffer_queue_),
382      false);
383
384  // Register callback method for the Android Simple Buffer Queue interface.
385  // This method will be called when the native audio layer needs audio data.
386  RETURN_ON_ERROR((*simple_buffer_queue_)
387                      ->RegisterCallback(simple_buffer_queue_,
388                                         SimpleBufferQueueCallback, this),
389                  false);
390
391  // Get the SLVolumeItf interface on the audio player.
392  RETURN_ON_ERROR(player_object_->GetInterface(player_object_.Get(),
393                                               SL_IID_VOLUME, &volume_),
394                  false);
395
396  // TODO(henrika): might not be required to set volume to max here since it
397  // seems to be default on most devices. Might be required for unit tests.
398  // RETURN_ON_ERROR((*volume_)->SetVolumeLevel(volume_, 0), false);
399
400  return true;
401}
402
403void OpenSLESPlayer::DestroyAudioPlayer() {
404  ALOGD("DestroyAudioPlayer");
405  RTC_DCHECK(thread_checker_.CalledOnValidThread());
406  if (!player_object_.Get())
407    return;
408  player_object_.Reset();
409  player_ = nullptr;
410  simple_buffer_queue_ = nullptr;
411  volume_ = nullptr;
412}
413
414// static
415void OpenSLESPlayer::SimpleBufferQueueCallback(
416    SLAndroidSimpleBufferQueueItf caller,
417    void* context) {
418  OpenSLESPlayer* stream = reinterpret_cast<OpenSLESPlayer*>(context);
419  stream->FillBufferQueue();
420}
421
422void OpenSLESPlayer::FillBufferQueue() {
423  RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread());
424  SLuint32 state = GetPlayState();
425  if (state != SL_PLAYSTATE_PLAYING) {
426    ALOGW("Buffer callback in non-playing state!");
427    return;
428  }
429  EnqueuePlayoutData();
430}
431
432void OpenSLESPlayer::EnqueuePlayoutData() {
433  // Check delta time between two successive callbacks and provide a warning
434  // if it becomes very large.
435  // TODO(henrika): using 100ms as upper limit but this value is rather random.
436  const uint32_t current_time = rtc::Time();
437  const uint32_t diff = current_time - last_play_time_;
438  if (diff > 100) {
439    ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff);
440  }
441  last_play_time_ = current_time;
442  // Read audio data from the WebRTC source using the FineAudioBuffer object
443  // to adjust for differences in buffer size between WebRTC (10ms) and native
444  // OpenSL ES.
445  SLint8* audio_ptr = audio_buffers_[buffer_index_].get();
446  fine_buffer_->GetPlayoutData(audio_ptr);
447  // Enqueue the decoded audio buffer for playback.
448  SLresult err =
449      (*simple_buffer_queue_)
450          ->Enqueue(simple_buffer_queue_, audio_ptr, bytes_per_buffer_);
451  if (SL_RESULT_SUCCESS != err) {
452    ALOGE("Enqueue failed: %d", err);
453  }
454  buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
455}
456
457SLuint32 OpenSLESPlayer::GetPlayState() const {
458  RTC_DCHECK(player_);
459  SLuint32 state;
460  SLresult err = (*player_)->GetPlayState(player_, &state);
461  if (SL_RESULT_SUCCESS != err) {
462    ALOGE("GetPlayState failed: %d", err);
463  }
464  return state;
465}
466
467}  // namespace webrtc
468