19a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org/* 29a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 39a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * 49a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * Use of this source code is governed by a BSD-style license 59a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * that can be found in the LICENSE file in the root of the source 69a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * tree. An additional intellectual property rights grant can be found 79a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * in the file PATENTS. All contributing project authors may 89a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org * be found in the AUTHORS file in the root of the source tree. 99a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org */ 109a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 11e5abc854f3dc47de16067c2a41476c39b7626722henrik.lundin@webrtc.org#ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ 12e5abc854f3dc47de16067c2a41476c39b7626722henrik.lundin@webrtc.org#define WEBRTC_MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ 139a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 149a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org#include <assert.h> 159a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 16774b3d38a4a0f1a8ec08972a3c543cb5d607ce13henrike@webrtc.org#include "webrtc/base/constructormagic.h" 17e5abc854f3dc47de16067c2a41476c39b7626722henrik.lundin@webrtc.org#include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h" 189a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org#include "webrtc/system_wrappers/interface/scoped_ptr.h" 199a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org#include "webrtc/typedefs.h" 209a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 219a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.orgnamespace webrtc { 229a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 239a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org// Forward declarations. 249a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.orgclass BackgroundNoise; 259a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.orgclass RandomVector; 269a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.orgclass SyncBuffer; 279a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 289a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org// This class handles extrapolation of audio data from the sync_buffer to 299a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org// produce packet-loss concealment. 309a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org// TODO(hlundin): Refactor this class to divide the long methods into shorter 319a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org// ones. 329a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.orgclass Expand { 339a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org public: 349a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org Expand(BackgroundNoise* background_noise, 359a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org SyncBuffer* sync_buffer, 369a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org RandomVector* random_vector, 379a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int fs, 389a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org size_t num_channels) 39c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org : random_vector_(random_vector), 409a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org sync_buffer_(sync_buffer), 419a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org first_expand_(true), 429a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org fs_hz_(fs), 439a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org num_channels_(num_channels), 44c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org consecutive_expands_(0), 45c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org background_noise_(background_noise), 469a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org overlap_length_(5 * fs / 8000), 479a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org lag_index_direction_(0), 489a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org current_lag_index_(0), 499a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org stop_muting_(false), 509a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org channel_parameters_(new ChannelParameters[num_channels_]) { 519a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org assert(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000); 529a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org assert(fs <= kMaxSampleRate); // Should not be possible. 539a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org assert(num_channels_ > 0); 549a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org memset(expand_lags_, 0, sizeof(expand_lags_)); 559a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org Reset(); 569a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org } 579a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 589a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org virtual ~Expand() {} 599a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 609a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Resets the object. 61c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org virtual void Reset(); 629a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 639a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // The main method to produce concealment data. The data is appended to the 649a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // end of |output|. 65c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org virtual int Process(AudioMultiVector* output); 669a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 679a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Prepare the object to do extra expansion during normal operation following 689a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // a period of expands. 69c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org virtual void SetParametersForNormalAfterExpand(); 709a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 719a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Prepare the object to do extra expansion during merge operation following 729a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // a period of expands. 73c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org virtual void SetParametersForMergeAfterExpand(); 749a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 759a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Sets the mute factor for |channel| to |value|. 769a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org void SetMuteFactor(int16_t value, size_t channel) { 779a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org assert(channel < num_channels_); 789a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org channel_parameters_[channel].mute_factor = value; 799a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org } 809a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 819a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Returns the mute factor for |channel|. 829a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t MuteFactor(size_t channel) { 839a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org assert(channel < num_channels_); 849a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org return channel_parameters_[channel].mute_factor; 859a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org } 869a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 879a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Accessors and mutators. 88c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org virtual size_t overlap_length() const { return overlap_length_; } 899a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t max_lag() const { return max_lag_; } 909a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 91c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org protected: 92c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org static const int kMaxConsecutiveExpands = 200; 93c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org void GenerateRandomVector(int seed_increment, 94c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org size_t length, 95c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org int16_t* random_vector); 96c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 97c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org void GenerateBackgroundNoise(int16_t* random_vector, 98c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org size_t channel, 99c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org int16_t mute_slope, 100c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org bool too_many_expands, 101c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org size_t num_noise_samples, 102c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org int16_t* buffer); 103c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 104c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org // Initializes member variables at the beginning of an expand period. 105c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org void InitializeForAnExpandPeriod(); 106c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 107c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org bool TooManyExpands(); 108c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 109c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org // Analyzes the signal history in |sync_buffer_|, and set up all parameters 110c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org // necessary to produce concealment data. 111c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org void AnalyzeSignal(int16_t* random_vector); 112c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 113c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org RandomVector* random_vector_; 114c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org SyncBuffer* sync_buffer_; 115c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org bool first_expand_; 116c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org const int fs_hz_; 117c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org const size_t num_channels_; 118c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org int consecutive_expands_; 119c1caa69f05663fc729af5a921eb95a73709f7dcdturaj@webrtc.org 1209a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org private: 1219a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kUnvoicedLpcOrder = 6; 1229a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kNumCorrelationCandidates = 3; 1239a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kDistortionLength = 20; 1249a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kLpcAnalysisLength = 160; 1259a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kMaxSampleRate = 48000; 1269a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org static const int kNumLags = 3; 1279a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 1289a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org struct ChannelParameters { 1299a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Constructor. 1309a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org ChannelParameters() 1319a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org : mute_factor(16384), 1329a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org ar_gain(0), 1339a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org ar_gain_scale(0), 1349a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org voice_mix_factor(0), 1359a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org current_voice_mix_factor(0), 1369a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org onset(false), 1379a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org mute_slope(0) { 1389a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org memset(ar_filter, 0, sizeof(ar_filter)); 1399a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org memset(ar_filter_state, 0, sizeof(ar_filter_state)); 1409a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org } 1419a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t mute_factor; 1429a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t ar_filter[kUnvoicedLpcOrder + 1]; 1439a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t ar_filter_state[kUnvoicedLpcOrder]; 1449a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t ar_gain; 1459a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t ar_gain_scale; 1469a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t voice_mix_factor; /* Q14 */ 1479a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t current_voice_mix_factor; /* Q14 */ 1489b1b52533c0e6accb1b0d3a4c1934856cd4473edhenrik.lundin@webrtc.org AudioVector expand_vector0; 1499b1b52533c0e6accb1b0d3a4c1934856cd4473edhenrik.lundin@webrtc.org AudioVector expand_vector1; 1509a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org bool onset; 1519a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t mute_slope; /* Q20 */ 1529a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org }; 1539a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 1549a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // Calculate the auto-correlation of |input|, with length |input_length| 1559a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // samples. The correlation is calculated from a downsampled version of 1569a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // |input|, and is written to |output|. The scale factor is written to 1579a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org // |output_scale|. Returns the length of the correlation vector. 158045e45efeca8776975254550137ec65268aadb54turaj@webrtc.org int16_t Correlation(const int16_t* input, size_t input_length, 1599a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t* output, int16_t* output_scale) const; 1609a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 1619a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org void UpdateLagIndex(); 1629a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 1639a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org BackgroundNoise* background_noise_; 164d1ef2b16a797ea74cc5ccb64598d41debc7a8413henrik.lundin@webrtc.org const size_t overlap_length_; 1659a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int16_t max_lag_; 1669a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org size_t expand_lags_[kNumLags]; 1679a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int lag_index_direction_; 1689a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org int current_lag_index_; 1699a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org bool stop_muting_; 170ba47616ee5a8d8a4d94e160b64b79a56845e291bandrew@webrtc.org scoped_ptr<ChannelParameters[]> channel_parameters_; 1719a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 1729a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org DISALLOW_COPY_AND_ASSIGN(Expand); 1739a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org}; 1749a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org 17537fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.orgstruct ExpandFactory { 17637fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org ExpandFactory() {} 17737fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org virtual ~ExpandFactory() {} 17837fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org 17937fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org virtual Expand* Create(BackgroundNoise* background_noise, 18037fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org SyncBuffer* sync_buffer, 18137fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org RandomVector* random_vector, 18237fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org int fs, 18337fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org size_t num_channels) const; 18437fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org}; 18537fb66d3bde95c19f5c80ece459f5dd632077ff5henrik.lundin@webrtc.org 1869a400812ca0006d12e538d465ab6728a8ecd07aahenrik.lundin@webrtc.org} // namespace webrtc 187e5abc854f3dc47de16067c2a41476c39b7626722henrik.lundin@webrtc.org#endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_EXPAND_H_ 188