1b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika/* 2b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * 4b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * Use of this source code is governed by a BSD-style license 5b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * that can be found in the LICENSE file in the root of the source 6b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * tree. An additional intellectual property rights grant can be found 7b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * in the file PATENTS. All contributing project authors may 8b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika * be found in the AUTHORS file in the root of the source tree. 9b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika */ 10b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 11b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#ifndef WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ 12b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#define WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ 13b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 14b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include <SLES/OpenSLES.h> 15b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include <SLES/OpenSLES_Android.h> 16b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include <SLES/OpenSLES_AndroidConfiguration.h> 17b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 18b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/base/scoped_ptr.h" 19b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/base/thread_checker.h" 20b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/modules/audio_device/android/audio_common.h" 21b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/modules/audio_device/android/audio_manager.h" 22b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/modules/audio_device/android/opensles_common.h" 23b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/modules/audio_device/include/audio_device_defines.h" 24b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#include "webrtc/modules/audio_device/audio_device_generic.h" 25ff761fba8274d93bd73e76c8b8a1f2d0776dd840Henrik Kjellander#include "webrtc/modules/utility/include/helpers_android.h" 26b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 27b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrikanamespace webrtc { 28b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 29b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrikaclass FineAudioBuffer; 30b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 31b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// Implements 16-bit mono PCM audio output support for Android using the 32b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// C based OpenSL ES API. No calls from C/C++ to Java using JNI is done. 33b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// 34b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// An instance must be created and destroyed on one and the same thread. 35b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// All public methods must also be called on the same thread. A thread checker 3691d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg// will RTC_DCHECK if any method is called on an invalid thread. Decoded audio 37b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// buffers are requested on a dedicated internal thread managed by the OpenSL 38b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// ES layer. 39b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// 40b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// The existing design forces the user to call InitPlayout() after Stoplayout() 41b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// to be able to call StartPlayout() again. This is inline with how the Java- 42b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// based implementation works. 43b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// 44b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// OpenSL ES is a native C API which have no Dalvik-related overhead such as 45b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// garbage collection pauses and it supports reduced audio output latency. 46b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// If the device doesn't claim this feature but supports API level 9 (Android 47b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// platform version 2.3) or later, then we can still use the OpenSL ES APIs but 48b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika// the output latency may be higher. 49b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrikaclass OpenSLESPlayer { 50b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika public: 51b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // The lower output latency path is used only if the application requests a 52b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // buffer count of 2 or more, and a buffer size and sample rate that are 53b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // compatible with the device's native output configuration provided via the 54b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // audio manager at construction. 55e71b24e42163afb48f24832ed079a92ee83f01b8henrika static const int kNumOfOpenSLESBuffers = 4; 56b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 57b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // There is no need for this class to use JNI. 58b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika static int32_t SetAndroidAudioDeviceObjects(void* javaVM, void* context) { 59b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika return 0; 60b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika } 61b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika static void ClearAndroidAudioDeviceObjects() {} 62b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 63b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika explicit OpenSLESPlayer(AudioManager* audio_manager); 64b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika ~OpenSLESPlayer(); 65b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 66b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int Init(); 67b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int Terminate(); 68b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 69b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int InitPlayout(); 70b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool PlayoutIsInitialized() const { return initialized_; } 71b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 72b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int StartPlayout(); 73b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int StopPlayout(); 74b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool Playing() const { return playing_; } 75b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 76b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int SpeakerVolumeIsAvailable(bool& available); 77b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int SetSpeakerVolume(uint32_t volume); 78b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int SpeakerVolume(uint32_t& volume) const; 79b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int MaxSpeakerVolume(uint32_t& maxVolume) const; 80b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int MinSpeakerVolume(uint32_t& minVolume) const; 81b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 82b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); 83b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 84b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika private: 85b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // These callback methods are called when data is required for playout. 86b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // They are both called from an internal "OpenSL ES thread" which is not 87b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // attached to the Dalvik VM. 88b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika static void SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller, 89b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void* context); 90b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void FillBufferQueue(); 91b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Reads audio data in PCM format using the AudioDeviceBuffer. 92b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Can be called both on the main thread (during Start()) and from the 93b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // internal audio thread while output streaming is active. 94b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void EnqueuePlayoutData(); 95b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 96b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Configures the SL_DATAFORMAT_PCM structure. 976955870806624479723addfae6dcf5d13968796cPeter Kasting SLDataFormat_PCM CreatePCMConfiguration(size_t channels, 98b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int sample_rate, 991380e266ff48be9718ce0867cfd65058cb09c5fcPeter Kasting size_t bits_per_sample); 100b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 101b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Allocate memory for audio buffers which will be used to render audio 102b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // via the SLAndroidSimpleBufferQueueItf interface. 103b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void AllocateDataBuffers(); 104b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 105b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Creates/destroys the main engine object and the SLEngineItf interface. 106b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool CreateEngine(); 107b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void DestroyEngine(); 108b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 109b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Creates/destroys the output mix object. 110b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool CreateMix(); 111b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void DestroyMix(); 112b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 113b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Creates/destroys the audio player and the simple-buffer object. 114b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Also creates the volume object. 115b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool CreateAudioPlayer(); 116b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika void DestroyAudioPlayer(); 117b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 118b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLuint32 GetPlayState() const; 119b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 120b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Ensures that methods are called from the same thread as this object is 121b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // created on. 122b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika rtc::ThreadChecker thread_checker_; 123b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 124b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Stores thread ID in first call to SimpleBufferQueueCallback() from internal 125b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // non-application thread which is not attached to the Dalvik JVM. 126b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Detached during construction of this object. 127b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika rtc::ThreadChecker thread_checker_opensles_; 128b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 129b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Contains audio parameters provided to this class at construction by the 130b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // AudioManager. 131b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika const AudioParameters audio_parameters_; 132b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 133b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the 134b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // AudioDeviceModuleImpl class and called by AudioDeviceModuleImpl::Create(). 135b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika AudioDeviceBuffer* audio_device_buffer_; 136b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 137b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool initialized_; 138b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika bool playing_; 139b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 140b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // PCM-type format definition. 141b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // TODO(henrika): add support for SLAndroidDataFormat_PCM_EX (android-21) if 142b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // 32-bit float representation is needed. 143b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLDataFormat_PCM pcm_format_; 144b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 145b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Number of bytes per audio buffer in each |audio_buffers_[i]|. 146b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Typical sizes are 480 or 512 bytes corresponding to native output buffer 147b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // sizes of 240 or 256 audio frames respectively. 1481380e266ff48be9718ce0867cfd65058cb09c5fcPeter Kasting size_t bytes_per_buffer_; 149b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 150b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Queue of audio buffers to be used by the player object for rendering 151b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // audio. They will be used in a Round-robin way and the size of each buffer 152b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // is given by FineAudioBuffer::RequiredBufferSizeBytes(). 153b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika rtc::scoped_ptr<SLint8[]> audio_buffers_[kNumOfOpenSLESBuffers]; 154b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 155b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data 156b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // in chunks of 10ms. It then allows for this data to be pulled in 157b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // a finer or coarser granularity. I.e. interacting with this class instead 158b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // of directly with the AudioDeviceBuffer one can ask for any number of 159b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // audio data samples. 160b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Example: native buffer size is 240 audio frames at 48kHz sample rate. 161b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // WebRTC will provide 480 audio frames per 10ms but OpenSL ES asks for 240 162b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // in each callback (one every 5ms). This class can then ask for 240 and the 163b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // FineAudioBuffer will ask WebRTC for new data only every second callback 164b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // and also cach non-utilized audio. 165b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika rtc::scoped_ptr<FineAudioBuffer> fine_buffer_; 166b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 167b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Keeps track of active audio buffer 'n' in the audio_buffers_[n] queue. 168b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... 169b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika int buffer_index_; 170b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 171b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // The engine object which provides the SLEngineItf interface. 172b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Created by the global Open SL ES constructor slCreateEngine(). 173b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika webrtc::ScopedSLObjectItf engine_object_; 174b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 175b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // This interface exposes creation methods for all the OpenSL ES object types. 176b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // It is the OpenSL ES API entry point. 177b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLEngineItf engine_; 178b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 179b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // Output mix object to be used by the player object. 180b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika webrtc::ScopedSLObjectItf output_mix_; 181b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 182b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // The audio player media object plays out audio to the speakers. It also 183b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // supports volume control. 184b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika webrtc::ScopedSLObjectItf player_object_; 185b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 186b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // This interface is supported on the audio player and it controls the state 187b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // of the audio player. 188b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLPlayItf player_; 189b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 190b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // The Android Simple Buffer Queue interface is supported on the audio player 191b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // and it provides methods to send audio data from the source to the audio 192b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // player for rendering. 193b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLAndroidSimpleBufferQueueItf simple_buffer_queue_; 194b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 195b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // This interface exposes controls for manipulating the object’s audio volume 196b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika // properties. This interface is supported on the Audio Player object. 197b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika SLVolumeItf volume_; 198e71b24e42163afb48f24832ed079a92ee83f01b8henrika 199e71b24e42163afb48f24832ed079a92ee83f01b8henrika // Last time the OpenSL ES layer asked for audio data to play out. 200e71b24e42163afb48f24832ed079a92ee83f01b8henrika uint32_t last_play_time_; 201b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika}; 202b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 203b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika} // namespace webrtc 204b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika 205b26198972c1fcb4aa7abaf3895b007e301e7d5dchenrika#endif // WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_OPENSLES_PLAYER_H_ 206