15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)/* 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * Use of this source code is governed by a BSD-style license 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * that can be found in the LICENSE file in the root of the source 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * tree. An additional intellectual property rights grant can be found 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * in the file PATENTS. All contributing project authors may 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) * be found in the AUTHORS file in the root of the source tree. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) */ 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#ifndef WEBRTC_MODULES_AUDIO_CONFERENCE_MIXER_SOURCE_AUDIO_CONFERENCE_MIXER_IMPL_H_ 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#define WEBRTC_MODULES_AUDIO_CONFERENCE_MIXER_SOURCE_AUDIO_CONFERENCE_MIXER_IMPL_H_ 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <list> 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include <map> 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/engine_configurations.h" 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer.h" 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/modules/audio_conference_mixer/source/level_indicator.h" 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/modules/audio_conference_mixer/source/memory_pool.h" 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/modules/audio_conference_mixer/source/time_scheduler.h" 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/modules/interface/module_common_types.h" 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)#include "webrtc/system_wrappers/interface/scoped_ptr.h" 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)namespace webrtc { 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class AudioProcessing; 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class CriticalSectionWrapper; 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)typedef std::list<AudioFrame*> AudioFrameList; 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)typedef std::list<MixerParticipant*> MixerParticipantList; 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)// Cheshire cat implementation of MixerParticipant's non virtual functions. 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class MixHistory 3453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles){ 3553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)public: 3653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) MixHistory(); 3753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) ~MixHistory(); 3853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 3953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) // MixerParticipant function 4053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) int32_t IsMixed(bool& mixed) const; 4153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Sets wasMixed to true if the participant was mixed previous mix 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // iteration. 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t WasMixed(bool& wasMixed) const; 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Updates the mixed status. 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t SetIsMixed(const bool mixed); 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void ResetMixedStatus(); 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)private: 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool _isMixed; 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)}; 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class AudioConferenceMixerImpl : public AudioConferenceMixer 555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles){ 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)public: 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // AudioProcessing only accepts 10 ms frames. 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) enum {kProcessPeriodicityInMs = 10}; 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioConferenceMixerImpl(int id); 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ~AudioConferenceMixerImpl(); 6293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Must be called after ctor. 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool Init(); 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Module functions 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t ChangeUniqueId(const int32_t id) OVERRIDE; 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t TimeUntilNextProcess() OVERRIDE; 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t Process() OVERRIDE; 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // AudioConferenceMixer functions 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t RegisterMixedStreamCallback( 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioMixerOutputReceiver& mixReceiver) OVERRIDE; 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t UnRegisterMixedStreamCallback() OVERRIDE; 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t RegisterMixerStatusCallback( 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioMixerStatusReceiver& mixerStatusCallback, 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const uint32_t amountOf10MsBetweenCallbacks) OVERRIDE; 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t UnRegisterMixerStatusCallback() OVERRIDE; 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t SetMixabilityStatus(MixerParticipant& participant, 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool mixable) OVERRIDE; 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t MixabilityStatus(MixerParticipant& participant, 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool& mixable) OVERRIDE; 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t SetMinimumMixingFrequency(Frequency freq) OVERRIDE; 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t SetAnonymousMixabilityStatus( 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipant& participant, const bool mixable) OVERRIDE; 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) virtual int32_t AnonymousMixabilityStatus( 875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipant& participant, bool& mixable) OVERRIDE; 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)private: 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) enum{DEFAULT_AUDIO_FRAME_POOLSIZE = 50}; 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Set/get mix frequency 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t SetOutputFrequency(const Frequency frequency); 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Frequency OutputFrequency() const; 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Fills mixList with the AudioFrames pointers that should be used when 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // mixing. Fills mixParticipantList with ParticipantStatistics for the 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // participants who's AudioFrames are inside mixList. 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // maxAudioFrameCounter both input and output specifies how many more 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // AudioFrames that are allowed to be mixed. 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // rampOutList contain AudioFrames corresponding to an audio stream that 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // used to be mixed but shouldn't be mixed any longer. These AudioFrames 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // should be ramped out over this AudioFrame to avoid audio discontinuities. 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void UpdateToMix( 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioFrameList* mixList, 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioFrameList* rampOutList, 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) std::map<int, MixerParticipant*>* mixParticipantList, 1075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t& maxAudioFrameCounter); 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return the lowest mixing frequency that can be used without having to 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // downsample any audio. 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t GetLowestMixingFrequency(); 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t GetLowestMixingFrequencyFromList(MixerParticipantList* mixList); 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Return the AudioFrames that should be mixed anonymously. 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void GetAdditionalAudio(AudioFrameList* additionalFramesList); 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Update the MixHistory of all MixerParticipants. mixedParticipantsList 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // should contain a map of MixerParticipants that have been mixed. 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void UpdateMixedStatus( 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) std::map<int, MixerParticipant*>& mixedParticipantsList); 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Clears audioFrameList and reclaims all memory associated with it. 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void ClearAudioFrameList(AudioFrameList* audioFrameList); 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Update the list of MixerParticipants who have a positive VAD. mixList 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // should be a list of AudioFrames 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) void UpdateVADPositiveParticipants( 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioFrameList* mixList); 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // This function returns true if it finds the MixerParticipant in the 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // specified list of MixerParticipants. 1325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool IsParticipantInList( 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipant& participant, 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipantList* participantList) const; 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Add/remove the MixerParticipant to the specified 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // MixerParticipant list. 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool AddParticipantToList( 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipant& participant, 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipantList* participantList); 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) bool RemoveParticipantFromList( 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipant& removeParticipant, 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MixerParticipantList* participantList); 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Mix the AudioFrames stored in audioFrameList into mixedAudio. 1465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) int32_t MixFromList( 1475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) AudioFrame& mixedAudio, 1485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) const AudioFrameList* audioFrameList); 1495267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // Mix the AudioFrames stored in audioFrameList into mixedAudio. No 1505267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // record will be kept of this mix (e.g. the corresponding MixerParticipants 1515267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // will not be marked as IsMixed() 1525267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) int32_t MixAnonomouslyFromList(AudioFrame& mixedAudio, 1535267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) const AudioFrameList* audioFrameList); 1545267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1555267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) bool LimitMixedAudio(AudioFrame& mixedAudio); 1565267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) 1575267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) // Scratch memory 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Note that the scratch memory may only be touched in the scope of 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) // Process(). 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) size_t _scratchParticipantsToMixAmount; 161 ParticipantStatistics _scratchMixedParticipants[ 162 kMaximumAmountOfMixedParticipants]; 163 uint32_t _scratchVadPositiveParticipantsAmount; 164 ParticipantStatistics _scratchVadPositiveParticipants[ 165 kMaximumAmountOfMixedParticipants]; 166 167 scoped_ptr<CriticalSectionWrapper> _crit; 168 scoped_ptr<CriticalSectionWrapper> _cbCrit; 169 170 int32_t _id; 171 172 Frequency _minimumMixingFreq; 173 174 // Mix result callback 175 AudioMixerOutputReceiver* _mixReceiver; 176 177 AudioMixerStatusReceiver* _mixerStatusCallback; 178 uint32_t _amountOf10MsBetweenCallbacks; 179 uint32_t _amountOf10MsUntilNextCallback; 180 bool _mixerStatusCb; 181 182 // The current sample frequency and sample size when mixing. 183 Frequency _outputFrequency; 184 uint16_t _sampleSize; 185 186 // Memory pool to avoid allocating/deallocating AudioFrames 187 MemoryPool<AudioFrame>* _audioFramePool; 188 189 // List of all participants. Note all lists are disjunct 190 MixerParticipantList _participantList; // May be mixed. 191 // Always mixed, anonomously. 192 MixerParticipantList _additionalParticipantList; 193 194 size_t _numMixedParticipants; 195 // Determines if we will use a limiter for clipping protection during 196 // mixing. 197 bool use_limiter_; 198 199 uint32_t _timeStamp; 200 201 // Metronome class. 202 TimeScheduler _timeScheduler; 203 204 // Smooth level indicator. 205 LevelIndicator _mixedAudioLevel; 206 207 // Counter keeping track of concurrent calls to process. 208 // Note: should never be higher than 1 or lower than 0. 209 int16_t _processCalls; 210 211 // Used for inhibiting saturation in mixing. 212 scoped_ptr<AudioProcessing> _limiter; 213}; 214} // namespace webrtc 215 216#endif // WEBRTC_MODULES_AUDIO_CONFERENCE_MIXER_SOURCE_AUDIO_CONFERENCE_MIXER_IMPL_H_ 217