neteq_impl.h revision 74640895fafbb90a6630a6a91b80da0a7cff229c
1/* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 12#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 13 14#include "webrtc/base/constructormagic.h" 15#include "webrtc/base/scoped_ptr.h" 16#include "webrtc/base/thread_annotations.h" 17#include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h" 18#include "webrtc/modules/audio_coding/neteq/defines.h" 19#include "webrtc/modules/audio_coding/neteq/include/neteq.h" 20#include "webrtc/modules/audio_coding/neteq/packet.h" // Declare PacketList. 21#include "webrtc/modules/audio_coding/neteq/random_vector.h" 22#include "webrtc/modules/audio_coding/neteq/rtcp.h" 23#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" 24#include "webrtc/typedefs.h" 25 26namespace webrtc { 27 28// Forward declarations. 29class Accelerate; 30class BackgroundNoise; 31class BufferLevelFilter; 32class ComfortNoise; 33class CriticalSectionWrapper; 34class DecisionLogic; 35class DecoderDatabase; 36class DelayManager; 37class DelayPeakDetector; 38class DtmfBuffer; 39class DtmfToneGenerator; 40class Expand; 41class Merge; 42class Normal; 43class PacketBuffer; 44class PayloadSplitter; 45class PostDecodeVad; 46class PreemptiveExpand; 47class RandomVector; 48class SyncBuffer; 49class TimestampScaler; 50struct AccelerateFactory; 51struct DtmfEvent; 52struct ExpandFactory; 53struct PreemptiveExpandFactory; 54 55class NetEqImpl : public webrtc::NetEq { 56 public: 57 // Creates a new NetEqImpl object. The object will assume ownership of all 58 // injected dependencies, and will delete them when done. 59 NetEqImpl(const NetEq::Config& config, 60 BufferLevelFilter* buffer_level_filter, 61 DecoderDatabase* decoder_database, 62 DelayManager* delay_manager, 63 DelayPeakDetector* delay_peak_detector, 64 DtmfBuffer* dtmf_buffer, 65 DtmfToneGenerator* dtmf_tone_generator, 66 PacketBuffer* packet_buffer, 67 PayloadSplitter* payload_splitter, 68 TimestampScaler* timestamp_scaler, 69 AccelerateFactory* accelerate_factory, 70 ExpandFactory* expand_factory, 71 PreemptiveExpandFactory* preemptive_expand_factory, 72 bool create_components = true); 73 74 ~NetEqImpl() override; 75 76 // Inserts a new packet into NetEq. The |receive_timestamp| is an indication 77 // of the time when the packet was received, and should be measured with 78 // the same tick rate as the RTP timestamp of the current payload. 79 // Returns 0 on success, -1 on failure. 80 int InsertPacket(const WebRtcRTPHeader& rtp_header, 81 const uint8_t* payload, 82 size_t length_bytes, 83 uint32_t receive_timestamp) override; 84 85 // Inserts a sync-packet into packet queue. Sync-packets are decoded to 86 // silence and are intended to keep AV-sync intact in an event of long packet 87 // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq 88 // might insert sync-packet when they observe that buffer level of NetEq is 89 // decreasing below a certain threshold, defined by the application. 90 // Sync-packets should have the same payload type as the last audio payload 91 // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change 92 // can be implied by inserting a sync-packet. 93 // Returns kOk on success, kFail on failure. 94 int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, 95 uint32_t receive_timestamp) override; 96 97 // Instructs NetEq to deliver 10 ms of audio data. The data is written to 98 // |output_audio|, which can hold (at least) |max_length| elements. 99 // The number of channels that were written to the output is provided in 100 // the output variable |num_channels|, and each channel contains 101 // |samples_per_channel| elements. If more than one channel is written, 102 // the samples are interleaved. 103 // The speech type is written to |type|, if |type| is not NULL. 104 // Returns kOK on success, or kFail in case of an error. 105 int GetAudio(size_t max_length, 106 int16_t* output_audio, 107 size_t* samples_per_channel, 108 int* num_channels, 109 NetEqOutputType* type) override; 110 111 // Associates |rtp_payload_type| with |codec| and stores the information in 112 // the codec database. Returns kOK on success, kFail on failure. 113 int RegisterPayloadType(enum NetEqDecoder codec, 114 uint8_t rtp_payload_type) override; 115 116 // Provides an externally created decoder object |decoder| to insert in the 117 // decoder database. The decoder implements a decoder of type |codec| and 118 // associates it with |rtp_payload_type|. The decoder will produce samples 119 // at the rate |sample_rate_hz|. Returns kOK on success, kFail on failure. 120 int RegisterExternalDecoder(AudioDecoder* decoder, 121 enum NetEqDecoder codec, 122 uint8_t rtp_payload_type, 123 int sample_rate_hz) override; 124 125 // Removes |rtp_payload_type| from the codec database. Returns 0 on success, 126 // -1 on failure. 127 int RemovePayloadType(uint8_t rtp_payload_type) override; 128 129 bool SetMinimumDelay(int delay_ms) override; 130 131 bool SetMaximumDelay(int delay_ms) override; 132 133 int LeastRequiredDelayMs() const override; 134 135 int SetTargetDelay() override; 136 137 int TargetDelay() override; 138 139 int CurrentDelayMs() const override; 140 141 // Sets the playout mode to |mode|. 142 // Deprecated. 143 // TODO(henrik.lundin) Delete. 144 void SetPlayoutMode(NetEqPlayoutMode mode) override; 145 146 // Returns the current playout mode. 147 // Deprecated. 148 // TODO(henrik.lundin) Delete. 149 NetEqPlayoutMode PlayoutMode() const override; 150 151 // Writes the current network statistics to |stats|. The statistics are reset 152 // after the call. 153 int NetworkStatistics(NetEqNetworkStatistics* stats) override; 154 155 // Writes the current RTCP statistics to |stats|. The statistics are reset 156 // and a new report period is started with the call. 157 void GetRtcpStatistics(RtcpStatistics* stats) override; 158 159 // Same as RtcpStatistics(), but does not reset anything. 160 void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override; 161 162 // Enables post-decode VAD. When enabled, GetAudio() will return 163 // kOutputVADPassive when the signal contains no speech. 164 void EnableVad() override; 165 166 // Disables post-decode VAD. 167 void DisableVad() override; 168 169 bool GetPlayoutTimestamp(uint32_t* timestamp) override; 170 171 int SetTargetNumberOfChannels() override; 172 173 int SetTargetSampleRate() override; 174 175 // Returns the error code for the last occurred error. If no error has 176 // occurred, 0 is returned. 177 int LastError() const override; 178 179 // Returns the error code last returned by a decoder (audio or comfort noise). 180 // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check 181 // this method to get the decoder's error code. 182 int LastDecoderError() override; 183 184 // Flushes both the packet buffer and the sync buffer. 185 void FlushBuffers() override; 186 187 void PacketBufferStatistics(int* current_num_packets, 188 int* max_num_packets) const override; 189 190 // Get sequence number and timestamp of the latest RTP. 191 // This method is to facilitate NACK. 192 int DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const override; 193 194 // This accessor method is only intended for testing purposes. 195 const SyncBuffer* sync_buffer_for_test() const; 196 197 protected: 198 static const int kOutputSizeMs = 10; 199 static const size_t kMaxFrameSize = 2880; // 60 ms @ 48 kHz. 200 // TODO(hlundin): Provide a better value for kSyncBufferSize. 201 static const size_t kSyncBufferSize = 2 * kMaxFrameSize; 202 203 // Inserts a new packet into NetEq. This is used by the InsertPacket method 204 // above. Returns 0 on success, otherwise an error code. 205 // TODO(hlundin): Merge this with InsertPacket above? 206 int InsertPacketInternal(const WebRtcRTPHeader& rtp_header, 207 const uint8_t* payload, 208 size_t length_bytes, 209 uint32_t receive_timestamp, 210 bool is_sync_packet) 211 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 212 213 // Delivers 10 ms of audio data. The data is written to |output|, which can 214 // hold (at least) |max_length| elements. The number of channels that were 215 // written to the output is provided in the output variable |num_channels|, 216 // and each channel contains |samples_per_channel| elements. If more than one 217 // channel is written, the samples are interleaved. 218 // Returns 0 on success, otherwise an error code. 219 int GetAudioInternal(size_t max_length, 220 int16_t* output, 221 size_t* samples_per_channel, 222 int* num_channels) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 223 224 // Provides a decision to the GetAudioInternal method. The decision what to 225 // do is written to |operation|. Packets to decode are written to 226 // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When 227 // DTMF should be played, |play_dtmf| is set to true by the method. 228 // Returns 0 on success, otherwise an error code. 229 int GetDecision(Operations* operation, 230 PacketList* packet_list, 231 DtmfEvent* dtmf_event, 232 bool* play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 233 234 // Decodes the speech packets in |packet_list|, and writes the results to 235 // |decoded_buffer|, which is allocated to hold |decoded_buffer_length| 236 // elements. The length of the decoded data is written to |decoded_length|. 237 // The speech type -- speech or (codec-internal) comfort noise -- is written 238 // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389 239 // comfort noise, those are not decoded. 240 int Decode(PacketList* packet_list, 241 Operations* operation, 242 int* decoded_length, 243 AudioDecoder::SpeechType* speech_type) 244 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 245 246 // Sub-method to Decode(). Performs codec internal CNG. 247 int DecodeCng(AudioDecoder* decoder, int* decoded_length, 248 AudioDecoder::SpeechType* speech_type) 249 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 250 251 // Sub-method to Decode(). Performs the actual decoding. 252 int DecodeLoop(PacketList* packet_list, 253 const Operations& operation, 254 AudioDecoder* decoder, 255 int* decoded_length, 256 AudioDecoder::SpeechType* speech_type) 257 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 258 259 // Sub-method which calls the Normal class to perform the normal operation. 260 void DoNormal(const int16_t* decoded_buffer, 261 size_t decoded_length, 262 AudioDecoder::SpeechType speech_type, 263 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 264 265 // Sub-method which calls the Merge class to perform the merge operation. 266 void DoMerge(int16_t* decoded_buffer, 267 size_t decoded_length, 268 AudioDecoder::SpeechType speech_type, 269 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 270 271 // Sub-method which calls the Expand class to perform the expand operation. 272 int DoExpand(bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 273 274 // Sub-method which calls the Accelerate class to perform the accelerate 275 // operation. 276 int DoAccelerate(int16_t* decoded_buffer, 277 size_t decoded_length, 278 AudioDecoder::SpeechType speech_type, 279 bool play_dtmf, 280 bool fast_accelerate) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 281 282 // Sub-method which calls the PreemptiveExpand class to perform the 283 // preemtive expand operation. 284 int DoPreemptiveExpand(int16_t* decoded_buffer, 285 size_t decoded_length, 286 AudioDecoder::SpeechType speech_type, 287 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 288 289 // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort 290 // noise. |packet_list| can either contain one SID frame to update the 291 // noise parameters, or no payload at all, in which case the previously 292 // received parameters are used. 293 int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) 294 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 295 296 // Calls the audio decoder to generate codec-internal comfort noise when 297 // no packet was received. 298 void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length) 299 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 300 301 // Calls the DtmfToneGenerator class to generate DTMF tones. 302 int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) 303 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 304 305 // Produces packet-loss concealment using alternative methods. If the codec 306 // has an internal PLC, it is called to generate samples. Otherwise, the 307 // method performs zero-stuffing. 308 void DoAlternativePlc(bool increase_timestamp) 309 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 310 311 // Overdub DTMF on top of |output|. 312 int DtmfOverdub(const DtmfEvent& dtmf_event, 313 size_t num_channels, 314 int16_t* output) const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 315 316 // Extracts packets from |packet_buffer_| to produce at least 317 // |required_samples| samples. The packets are inserted into |packet_list|. 318 // Returns the number of samples that the packets in the list will produce, or 319 // -1 in case of an error. 320 int ExtractPackets(size_t required_samples, PacketList* packet_list) 321 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 322 323 // Resets various variables and objects to new values based on the sample rate 324 // |fs_hz| and |channels| number audio channels. 325 void SetSampleRateAndChannels(int fs_hz, size_t channels) 326 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 327 328 // Returns the output type for the audio produced by the latest call to 329 // GetAudio(). 330 NetEqOutputType LastOutputType() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 331 332 // Updates Expand and Merge. 333 virtual void UpdatePlcComponents(int fs_hz, size_t channels) 334 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 335 336 // Creates DecisionLogic object with the mode given by |playout_mode_|. 337 virtual void CreateDecisionLogic() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 338 339 const rtc::scoped_ptr<CriticalSectionWrapper> crit_sect_; 340 const rtc::scoped_ptr<BufferLevelFilter> buffer_level_filter_ 341 GUARDED_BY(crit_sect_); 342 const rtc::scoped_ptr<DecoderDatabase> decoder_database_ 343 GUARDED_BY(crit_sect_); 344 const rtc::scoped_ptr<DelayManager> delay_manager_ GUARDED_BY(crit_sect_); 345 const rtc::scoped_ptr<DelayPeakDetector> delay_peak_detector_ 346 GUARDED_BY(crit_sect_); 347 const rtc::scoped_ptr<DtmfBuffer> dtmf_buffer_ GUARDED_BY(crit_sect_); 348 const rtc::scoped_ptr<DtmfToneGenerator> dtmf_tone_generator_ 349 GUARDED_BY(crit_sect_); 350 const rtc::scoped_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_); 351 const rtc::scoped_ptr<PayloadSplitter> payload_splitter_ 352 GUARDED_BY(crit_sect_); 353 const rtc::scoped_ptr<TimestampScaler> timestamp_scaler_ 354 GUARDED_BY(crit_sect_); 355 const rtc::scoped_ptr<PostDecodeVad> vad_ GUARDED_BY(crit_sect_); 356 const rtc::scoped_ptr<ExpandFactory> expand_factory_ GUARDED_BY(crit_sect_); 357 const rtc::scoped_ptr<AccelerateFactory> accelerate_factory_ 358 GUARDED_BY(crit_sect_); 359 const rtc::scoped_ptr<PreemptiveExpandFactory> preemptive_expand_factory_ 360 GUARDED_BY(crit_sect_); 361 362 rtc::scoped_ptr<BackgroundNoise> background_noise_ GUARDED_BY(crit_sect_); 363 rtc::scoped_ptr<DecisionLogic> decision_logic_ GUARDED_BY(crit_sect_); 364 rtc::scoped_ptr<AudioMultiVector> algorithm_buffer_ GUARDED_BY(crit_sect_); 365 rtc::scoped_ptr<SyncBuffer> sync_buffer_ GUARDED_BY(crit_sect_); 366 rtc::scoped_ptr<Expand> expand_ GUARDED_BY(crit_sect_); 367 rtc::scoped_ptr<Normal> normal_ GUARDED_BY(crit_sect_); 368 rtc::scoped_ptr<Merge> merge_ GUARDED_BY(crit_sect_); 369 rtc::scoped_ptr<Accelerate> accelerate_ GUARDED_BY(crit_sect_); 370 rtc::scoped_ptr<PreemptiveExpand> preemptive_expand_ GUARDED_BY(crit_sect_); 371 RandomVector random_vector_ GUARDED_BY(crit_sect_); 372 rtc::scoped_ptr<ComfortNoise> comfort_noise_ GUARDED_BY(crit_sect_); 373 Rtcp rtcp_ GUARDED_BY(crit_sect_); 374 StatisticsCalculator stats_ GUARDED_BY(crit_sect_); 375 int fs_hz_ GUARDED_BY(crit_sect_); 376 int fs_mult_ GUARDED_BY(crit_sect_); 377 size_t output_size_samples_ GUARDED_BY(crit_sect_); 378 size_t decoder_frame_length_ GUARDED_BY(crit_sect_); 379 Modes last_mode_ GUARDED_BY(crit_sect_); 380 rtc::scoped_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_); 381 size_t decoded_buffer_length_ GUARDED_BY(crit_sect_); 382 rtc::scoped_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_); 383 uint32_t playout_timestamp_ GUARDED_BY(crit_sect_); 384 bool new_codec_ GUARDED_BY(crit_sect_); 385 uint32_t timestamp_ GUARDED_BY(crit_sect_); 386 bool reset_decoder_ GUARDED_BY(crit_sect_); 387 uint8_t current_rtp_payload_type_ GUARDED_BY(crit_sect_); 388 uint8_t current_cng_rtp_payload_type_ GUARDED_BY(crit_sect_); 389 uint32_t ssrc_ GUARDED_BY(crit_sect_); 390 bool first_packet_ GUARDED_BY(crit_sect_); 391 int error_code_ GUARDED_BY(crit_sect_); // Store last error code. 392 int decoder_error_code_ GUARDED_BY(crit_sect_); 393 const BackgroundNoiseMode background_noise_mode_ GUARDED_BY(crit_sect_); 394 NetEqPlayoutMode playout_mode_ GUARDED_BY(crit_sect_); 395 bool enable_fast_accelerate_ GUARDED_BY(crit_sect_); 396 397 // These values are used by NACK module to estimate time-to-play of 398 // a missing packet. Occasionally, NetEq might decide to decode more 399 // than one packet. Therefore, these values store sequence number and 400 // timestamp of the first packet pulled from the packet buffer. In 401 // such cases, these values do not exactly represent the sequence number 402 // or timestamp associated with a 10ms audio pulled from NetEq. NACK 403 // module is designed to compensate for this. 404 int decoded_packet_sequence_number_ GUARDED_BY(crit_sect_); 405 uint32_t decoded_packet_timestamp_ GUARDED_BY(crit_sect_); 406 407 private: 408 RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl); 409}; 410 411} // namespace webrtc 412#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 413