1b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org/* 2b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. 3b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * 4b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * Use of this source code is governed by a BSD-style license 5b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * that can be found in the LICENSE file in the root of the source 6b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * tree. An additional intellectual property rights grant can be found 7b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * in the file PATENTS. All contributing project authors may 8b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * be found in the AUTHORS file in the root of the source tree. 9b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org */ 10b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 11b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org/* 12b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org * The core AEC algorithm, SSE2 version of speed-critical functions. 13b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org */ 14b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 15b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org#include <emmintrin.h> 16b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org#include <math.h> 17b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org#include <string.h> // memset 18b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 1999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" 20f8bddb4a17217a4cd8e92e4b322ee8df3327e1c6bjornv@webrtc.org#include "webrtc/modules/audio_processing/aec/aec_common.h" 213a4f3f1ce4f4576f8129bfe900d5cc4737936c4bbjornv@webrtc.org#include "webrtc/modules/audio_processing/aec/aec_core_internal.h" 223a4f3f1ce4f4576f8129bfe900d5cc4737936c4bbjornv@webrtc.org#include "webrtc/modules/audio_processing/aec/aec_rdft.h" 23b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 242f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org__inline static float MulRe(float aRe, float aIm, float bRe, float bIm) { 25b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return aRe * bRe - aIm * bIm; 26b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 27b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 282f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org__inline static float MulIm(float aRe, float aIm, float bRe, float bIm) { 29b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return aRe * bIm + aIm * bRe; 30b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 31b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 322f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.orgstatic void FilterFarSSE2(AecCore* aec, float yf[2][PART_LEN1]) { 33b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int i; 348ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org const int num_partitions = aec->num_partitions; 358ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org for (i = 0; i < num_partitions; i++) { 36b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int j; 37b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int xPos = (i + aec->xfBufBlockPos) * PART_LEN1; 38b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int pos = i * PART_LEN1; 39b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for wrap 408ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org if (i + aec->xfBufBlockPos >= num_partitions) { 412f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org xPos -= num_partitions * (PART_LEN1); 42b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 43b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 44b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // vectorized code (four at once) 45b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (j = 0; j + 3 < PART_LEN1; j += 4) { 46b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xfBuf_re = _mm_loadu_ps(&aec->xfBuf[0][xPos + j]); 47b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xfBuf_im = _mm_loadu_ps(&aec->xfBuf[1][xPos + j]); 48b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 wfBuf_re = _mm_loadu_ps(&aec->wfBuf[0][pos + j]); 49b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 wfBuf_im = _mm_loadu_ps(&aec->wfBuf[1][pos + j]); 50b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 yf_re = _mm_loadu_ps(&yf[0][j]); 51b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 yf_im = _mm_loadu_ps(&yf[1][j]); 52b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 a = _mm_mul_ps(xfBuf_re, wfBuf_re); 53b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 b = _mm_mul_ps(xfBuf_im, wfBuf_im); 54b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 c = _mm_mul_ps(xfBuf_re, wfBuf_im); 55b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 d = _mm_mul_ps(xfBuf_im, wfBuf_re); 56b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 e = _mm_sub_ps(a, b); 57b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 f = _mm_add_ps(c, d); 58b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 g = _mm_add_ps(yf_re, e); 59b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 h = _mm_add_ps(yf_im, f); 60b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&yf[0][j], g); 61b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&yf[1][j], h); 62b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 63b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // scalar code for the remaining items. 64b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (; j < PART_LEN1; j++) { 652f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org yf[0][j] += MulRe(aec->xfBuf[0][xPos + j], 662f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->xfBuf[1][xPos + j], 672f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->wfBuf[0][pos + j], 682f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->wfBuf[1][pos + j]); 692f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org yf[1][j] += MulIm(aec->xfBuf[0][xPos + j], 702f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->xfBuf[1][xPos + j], 712f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->wfBuf[0][pos + j], 722f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->wfBuf[1][pos + j]); 73b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 74b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 75b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 76b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 772f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.orgstatic void ScaleErrorSignalSSE2(AecCore* aec, float ef[2][PART_LEN1]) { 78b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 k1e_10f = _mm_set1_ps(1e-10f); 792f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 kMu = aec->extended_filter_enabled ? _mm_set1_ps(kExtendedMu) 802f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org : _mm_set1_ps(aec->normal_mu); 812f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 kThresh = aec->extended_filter_enabled 822f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org ? _mm_set1_ps(kExtendedErrorThreshold) 832f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org : _mm_set1_ps(aec->normal_error_threshold); 84b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 85b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int i; 86b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // vectorized code (four at once) 87b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (i = 0; i + 3 < PART_LEN1; i += 4) { 88b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xPow = _mm_loadu_ps(&aec->xPow[i]); 89b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_re_base = _mm_loadu_ps(&ef[0][i]); 90b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_im_base = _mm_loadu_ps(&ef[1][i]); 91b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 92b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xPowPlus = _mm_add_ps(xPow, k1e_10f); 93b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 ef_re = _mm_div_ps(ef_re_base, xPowPlus); 94b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 ef_im = _mm_div_ps(ef_im_base, xPowPlus); 95b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_re2 = _mm_mul_ps(ef_re, ef_re); 96b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_im2 = _mm_mul_ps(ef_im, ef_im); 97b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_sum2 = _mm_add_ps(ef_re2, ef_im2); 98b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 absEf = _mm_sqrt_ps(ef_sum2); 99b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 bigger = _mm_cmpgt_ps(absEf, kThresh); 100b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 absEfPlus = _mm_add_ps(absEf, k1e_10f); 101b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 absEfInv = _mm_div_ps(kThresh, absEfPlus); 102b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 ef_re_if = _mm_mul_ps(ef_re, absEfInv); 103b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 ef_im_if = _mm_mul_ps(ef_im, absEfInv); 104b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_re_if = _mm_and_ps(bigger, ef_re_if); 105b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_im_if = _mm_and_ps(bigger, ef_im_if); 106b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_re = _mm_andnot_ps(bigger, ef_re); 107b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_im = _mm_andnot_ps(bigger, ef_im); 108b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_re = _mm_or_ps(ef_re, ef_re_if); 109b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_im = _mm_or_ps(ef_im, ef_im_if); 110b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_re = _mm_mul_ps(ef_re, kMu); 111b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org ef_im = _mm_mul_ps(ef_im, kMu); 112b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 113b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&ef[0][i], ef_re); 114b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&ef[1][i], ef_im); 115b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 116b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // scalar code for the remaining items. 1178ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org { 1182f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const float mu = 1192f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org aec->extended_filter_enabled ? kExtendedMu : aec->normal_mu; 1202f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const float error_threshold = aec->extended_filter_enabled 1212f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org ? kExtendedErrorThreshold 1222f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org : aec->normal_error_threshold; 1238ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org for (; i < (PART_LEN1); i++) { 1242f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org float abs_ef; 1258ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[0][i] /= (aec->xPow[i] + 1e-10f); 1268ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[1][i] /= (aec->xPow[i] + 1e-10f); 1278ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org abs_ef = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]); 1288ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org 1298ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org if (abs_ef > error_threshold) { 1308ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org abs_ef = error_threshold / (abs_ef + 1e-10f); 1318ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[0][i] *= abs_ef; 1328ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[1][i] *= abs_ef; 1338ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org } 13499b6d9e8699a1fa767e0b053b6c513efc45481a7asapersson@webrtc.org 1358ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org // Stepsize factor 1368ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[0][i] *= mu; 1378ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org ef[1][i] *= mu; 1388ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org } 139b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 140b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 141b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 1422f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.orgstatic void FilterAdaptationSSE2(AecCore* aec, 1432f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org float* fft, 1442f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org float ef[2][PART_LEN1]) { 145b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int i, j; 1468ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org const int num_partitions = aec->num_partitions; 1478ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org for (i = 0; i < num_partitions; i++) { 1482f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org int xPos = (i + aec->xfBufBlockPos) * (PART_LEN1); 149b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int pos = i * PART_LEN1; 150b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Check for wrap 1518ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org if (i + aec->xfBufBlockPos >= num_partitions) { 1528ddec2c539a1daf4dd08607561090ac163cb9a7aandrew@webrtc.org xPos -= num_partitions * PART_LEN1; 153b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 154b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 155b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Process the whole array... 1562f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org for (j = 0; j < PART_LEN; j += 4) { 157b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Load xfBuf and ef. 158b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xfBuf_re = _mm_loadu_ps(&aec->xfBuf[0][xPos + j]); 159b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 xfBuf_im = _mm_loadu_ps(&aec->xfBuf[1][xPos + j]); 160b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_re = _mm_loadu_ps(&ef[0][j]); 161b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 ef_im = _mm_loadu_ps(&ef[1][j]); 162b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Calculate the product of conjugate(xfBuf) by ef. 163b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // re(conjugate(a) * b) = aRe * bRe + aIm * bIm 164b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // im(conjugate(a) * b)= aRe * bIm - aIm * bRe 165b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 a = _mm_mul_ps(xfBuf_re, ef_re); 166b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 b = _mm_mul_ps(xfBuf_im, ef_im); 167b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 c = _mm_mul_ps(xfBuf_re, ef_im); 168b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 d = _mm_mul_ps(xfBuf_im, ef_re); 169b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 e = _mm_add_ps(a, b); 170b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 f = _mm_sub_ps(c, d); 171b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Interleave real and imaginary parts. 172b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 g = _mm_unpacklo_ps(e, f); 173b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 h = _mm_unpackhi_ps(e, f); 174b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Store 1752f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_storeu_ps(&fft[2 * j + 0], g); 1762f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_storeu_ps(&fft[2 * j + 4], h); 177b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 178b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // ... and fixup the first imaginary entry. 179b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org fft[1] = MulRe(aec->xfBuf[0][xPos + PART_LEN], 180b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org -aec->xfBuf[1][xPos + PART_LEN], 1812f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org ef[0][PART_LEN], 1822f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org ef[1][PART_LEN]); 183b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 184b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org aec_rdft_inverse_128(fft); 1852f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org memset(fft + PART_LEN, 0, sizeof(float) * PART_LEN); 186b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 187b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // fft scaling 188b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 189b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float scale = 2.0f / PART_LEN2; 190b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 scale_ps = _mm_load_ps1(&scale); 1912f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org for (j = 0; j < PART_LEN; j += 4) { 192b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 fft_ps = _mm_loadu_ps(&fft[j]); 193b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 fft_scale = _mm_mul_ps(fft_ps, scale_ps); 194b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&fft[j], fft_scale); 195b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 196b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 197b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org aec_rdft_forward_128(fft); 198b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 199b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 200b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float wt1 = aec->wfBuf[1][pos]; 201b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org aec->wfBuf[0][pos + PART_LEN] += fft[1]; 2022f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org for (j = 0; j < PART_LEN; j += 4) { 203b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 wtBuf_re = _mm_loadu_ps(&aec->wfBuf[0][pos + j]); 204b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 wtBuf_im = _mm_loadu_ps(&aec->wfBuf[1][pos + j]); 205b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 fft0 = _mm_loadu_ps(&fft[2 * j + 0]); 206b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 fft4 = _mm_loadu_ps(&fft[2 * j + 4]); 2072f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 fft_re = 2082f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(2, 0, 2, 0)); 2092f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 fft_im = 2102f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(3, 1, 3, 1)); 211b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org wtBuf_re = _mm_add_ps(wtBuf_re, fft_re); 212b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org wtBuf_im = _mm_add_ps(wtBuf_im, fft_im); 213b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&aec->wfBuf[0][pos + j], wtBuf_re); 214b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&aec->wfBuf[1][pos + j], wtBuf_im); 215b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 216b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org aec->wfBuf[1][pos] = wt1; 217b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 218b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 219b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 220b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 2212f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.orgstatic __m128 mm_pow_ps(__m128 a, __m128 b) { 222b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // a^b = exp2(b * log2(a)) 223b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // exp2(x) and log2(x) are calculated using polynomial approximations. 224b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 log2_a, b_log2_a, a_exp_b; 225b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 226b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Calculate log2(x), x = a. 227b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 228b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // To calculate log2(x), we decompose x like this: 229b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // x = y * 2^n 230b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // n is an integer 231b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // y is in the [1.0, 2.0) range 232b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 233b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // log2(x) = log2(y) + n 234b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // n can be evaluated by playing with float representation. 235b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // log2(y) in a small range can be approximated, this code uses an order 236b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // five polynomial approximation. The coefficients have been 237b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // estimated with the Remez algorithm and the resulting 238b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // polynomial has a maximum relative error of 0.00086%. 239b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 240b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute n. 241b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // This is done by masking the exponent, shifting it into the top bit of 242b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // the mantissa, putting eight into the biased exponent (to shift/ 243b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // compensate the fact that the exponent has been shifted in the top/ 244b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // fractional part and finally getting rid of the implicit leading one 245b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // from the mantissa by substracting it out. 2462f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int float_exponent_mask[4] ALIGN16_END = { 2472f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0x7F800000, 0x7F800000, 0x7F800000, 0x7F800000}; 2482f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int eight_biased_exponent[4] ALIGN16_END = { 2492f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0x43800000, 0x43800000, 0x43800000, 0x43800000}; 2502f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int implicit_leading_one[4] ALIGN16_END = { 2512f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0x43BF8000, 0x43BF8000, 0x43BF8000, 0x43BF8000}; 252b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static const int shift_exponent_into_top_mantissa = 8; 2532f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 two_n = _mm_and_ps(a, *((__m128*)float_exponent_mask)); 2542f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 n_1 = _mm_castsi128_ps(_mm_srli_epi32( 2552f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_castps_si128(two_n), shift_exponent_into_top_mantissa)); 2562f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 n_0 = _mm_or_ps(n_1, *((__m128*)eight_biased_exponent)); 2572f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 n = _mm_sub_ps(n_0, *((__m128*)implicit_leading_one)); 258b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 259b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute y. 2602f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int mantissa_mask[4] ALIGN16_END = { 2612f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0x007FFFFF, 0x007FFFFF, 0x007FFFFF, 0x007FFFFF}; 2622f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int zero_biased_exponent_is_one[4] ALIGN16_END = { 2632f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0x3F800000, 0x3F800000, 0x3F800000, 0x3F800000}; 2642f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 mantissa = _mm_and_ps(a, *((__m128*)mantissa_mask)); 2652f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 y = 2662f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_or_ps(mantissa, *((__m128*)zero_biased_exponent_is_one)); 267b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 268b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Approximate log2(y) ~= (y - 1) * pol5(y). 269b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // pol5(y) = C5 * y^5 + C4 * y^4 + C3 * y^3 + C2 * y^2 + C1 * y + C0 2702f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END C5[4] = { 2712f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org -3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f}; 2722f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END 2732f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org C4[4] = {3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f}; 2742f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END 2752f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org C3[4] = {-1.2315303f, -1.2315303f, -1.2315303f, -1.2315303f}; 2762f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END 2772f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org C2[4] = {2.5988452f, 2.5988452f, 2.5988452f, 2.5988452f}; 2782f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END 2792f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org C1[4] = {-3.3241990f, -3.3241990f, -3.3241990f, -3.3241990f}; 2802f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float ALIGN16_END 2812f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org C0[4] = {3.1157899f, 3.1157899f, 3.1157899f, 3.1157899f}; 2822f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y_0 = _mm_mul_ps(y, *((__m128*)C5)); 2832f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y_1 = _mm_add_ps(pol5_y_0, *((__m128*)C4)); 284b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 pol5_y_2 = _mm_mul_ps(pol5_y_1, y); 2852f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y_3 = _mm_add_ps(pol5_y_2, *((__m128*)C3)); 286b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 pol5_y_4 = _mm_mul_ps(pol5_y_3, y); 2872f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y_5 = _mm_add_ps(pol5_y_4, *((__m128*)C2)); 288b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 pol5_y_6 = _mm_mul_ps(pol5_y_5, y); 2892f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y_7 = _mm_add_ps(pol5_y_6, *((__m128*)C1)); 290b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 pol5_y_8 = _mm_mul_ps(pol5_y_7, y); 2912f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 pol5_y = _mm_add_ps(pol5_y_8, *((__m128*)C0)); 2922f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 y_minus_one = 2932f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_sub_ps(y, *((__m128*)zero_biased_exponent_is_one)); 2942f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 log2_y = _mm_mul_ps(y_minus_one, pol5_y); 295b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 296b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Combine parts. 297b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org log2_a = _mm_add_ps(n, log2_y); 298b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 299b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 300b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // b * log2(a) 301b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org b_log2_a = _mm_mul_ps(b, log2_a); 302b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 303b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Calculate exp2(x), x = b * log2(a). 304b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 305b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // To calculate 2^x, we decompose x like this: 306b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // x = n + y 307b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // n is an integer, the value of x - 0.5 rounded down, therefore 308b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // y is in the [0.5, 1.5) range 309b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 310b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2^x = 2^n * 2^y 311b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2^n can be evaluated by playing with float representation. 312b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // 2^y in a small range can be approximated, this code uses an order two 313b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // polynomial approximation. The coefficients have been estimated 314b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // with the Remez algorithm and the resulting polynomial has a 315b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // maximum relative error of 0.17%. 316b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 317b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // To avoid over/underflow, we reduce the range of input to ]-127, 129]. 3182f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float max_input[4] ALIGN16_END = {129.f, 129.f, 3192f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 129.f, 129.f}; 3202f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float min_input[4] ALIGN16_END = { 3212f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org -126.99999f, -126.99999f, -126.99999f, -126.99999f}; 3222f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 x_min = _mm_min_ps(b_log2_a, *((__m128*)max_input)); 3232f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 x_max = _mm_max_ps(x_min, *((__m128*)min_input)); 324b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute n. 3252f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float half[4] ALIGN16_END = {0.5f, 0.5f, 3262f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 0.5f, 0.5f}; 3272f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 x_minus_half = _mm_sub_ps(x_max, *((__m128*)half)); 328b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128i x_minus_half_floor = _mm_cvtps_epi32(x_minus_half); 329b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute 2^n. 3302f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG int float_exponent_bias[4] ALIGN16_END = { 3312f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 127, 127, 127, 127}; 332b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org static const int float_exponent_shift = 23; 3332f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128i two_n_exponent = 3342f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_add_epi32(x_minus_half_floor, *((__m128i*)float_exponent_bias)); 3352f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 two_n = 3362f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_castsi128_ps(_mm_slli_epi32(two_n_exponent, float_exponent_shift)); 337b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Compute y. 338b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 y = _mm_sub_ps(x_max, _mm_cvtepi32_ps(x_minus_half_floor)); 339b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Approximate 2^y ~= C2 * y^2 + C1 * y + C0. 3402f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float C2[4] ALIGN16_END = { 3412f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f}; 3422f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float C1[4] ALIGN16_END = { 3432f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f}; 3442f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org static const ALIGN16_BEG float C0[4] ALIGN16_END = {1.0017247f, 1.0017247f, 3452f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org 1.0017247f, 1.0017247f}; 3462f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 exp2_y_0 = _mm_mul_ps(y, *((__m128*)C2)); 3472f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 exp2_y_1 = _mm_add_ps(exp2_y_0, *((__m128*)C1)); 348b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 exp2_y_2 = _mm_mul_ps(exp2_y_1, y); 3492f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 exp2_y = _mm_add_ps(exp2_y_2, *((__m128*)C0)); 350b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 351b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Combine parts. 352b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org a_exp_b = _mm_mul_ps(exp2_y, two_n); 353b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 354b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org return a_exp_b; 355b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 356b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 3572f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.orgstatic void OverdriveAndSuppressSSE2(AecCore* aec, 3582f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org float hNl[PART_LEN1], 359b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const float hNlFb, 360b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org float efw[2][PART_LEN1]) { 361b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org int i; 362b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_hNlFb = _mm_set1_ps(hNlFb); 363b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_one = _mm_set1_ps(1.0f); 364b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_minus_one = _mm_set1_ps(-1.0f); 365b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_overDriveSm = _mm_set1_ps(aec->overDriveSm); 366b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // vectorized code (four at once) 3672f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org for (i = 0; i + 3 < PART_LEN1; i += 4) { 368b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Weight subbands 369b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 vec_hNl = _mm_loadu_ps(&hNl[i]); 370b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_weightCurve = _mm_loadu_ps(&WebRtcAec_weightCurve[i]); 371b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 bigger = _mm_cmpgt_ps(vec_hNl, vec_hNlFb); 3722f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 vec_weightCurve_hNlFb = _mm_mul_ps(vec_weightCurve, vec_hNlFb); 373b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_one_weightCurve = _mm_sub_ps(vec_one, vec_weightCurve); 3742f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 vec_one_weightCurve_hNl = 3752f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_mul_ps(vec_one_weightCurve, vec_hNl); 376b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_if0 = _mm_andnot_ps(bigger, vec_hNl); 377b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org const __m128 vec_if1 = _mm_and_ps( 378b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org bigger, _mm_add_ps(vec_weightCurve_hNlFb, vec_one_weightCurve_hNl)); 379b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org vec_hNl = _mm_or_ps(vec_if0, vec_if1); 380b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 381b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 3822f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 vec_overDriveCurve = 3832f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_loadu_ps(&WebRtcAec_overDriveCurve[i]); 3842f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org const __m128 vec_overDriveSm_overDriveCurve = 3852f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org _mm_mul_ps(vec_overDriveSm, vec_overDriveCurve); 386b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org vec_hNl = mm_pow_ps(vec_hNl, vec_overDriveSm_overDriveCurve); 387b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&hNl[i], vec_hNl); 388b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 389b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 390b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Suppress error signal 391b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org { 392b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 vec_efw_re = _mm_loadu_ps(&efw[0][i]); 393b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org __m128 vec_efw_im = _mm_loadu_ps(&efw[1][i]); 394b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org vec_efw_re = _mm_mul_ps(vec_efw_re, vec_hNl); 395b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org vec_efw_im = _mm_mul_ps(vec_efw_im, vec_hNl); 396b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 397b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Ooura fft returns incorrect sign on imaginary component. It matters 398b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // here because we are making an additive change with comfort noise. 399b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org vec_efw_im = _mm_mul_ps(vec_efw_im, vec_minus_one); 400b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&efw[0][i], vec_efw_re); 401b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org _mm_storeu_ps(&efw[1][i], vec_efw_im); 402b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 403b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 404b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // scalar code for the remaining items. 405b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org for (; i < PART_LEN1; i++) { 406b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Weight subbands 407b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org if (hNl[i] > hNlFb) { 408b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org hNl[i] = WebRtcAec_weightCurve[i] * hNlFb + 4092f40af3ae55077b771fe301e19750984348d3eafandrew@webrtc.org (1 - WebRtcAec_weightCurve[i]) * hNl[i]; 410b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 411b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org hNl[i] = powf(hNl[i], aec->overDriveSm * WebRtcAec_overDriveCurve[i]); 412b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 413b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Suppress error signal 414b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org efw[0][i] *= hNl[i]; 415b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org efw[1][i] *= hNl[i]; 416b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 417b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // Ooura fft returns incorrect sign on imaginary component. It matters 418b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org // here because we are making an additive change with comfort noise. 419b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org efw[1][i] *= -1; 420b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org } 421b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 422b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org 42399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org__inline static void _mm_add_ps_4x1(__m128 sum, float *dst) { 42499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // A+B C+D 42599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org sum = _mm_add_ps(sum, _mm_shuffle_ps(sum, sum, _MM_SHUFFLE(0, 0, 3, 2))); 42699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // A+B+C+D A+B+C+D 42799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org sum = _mm_add_ps(sum, _mm_shuffle_ps(sum, sum, _MM_SHUFFLE(1, 1, 1, 1))); 42899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_store_ss(dst, sum); 42999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 43099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.orgstatic int PartitionDelay(const AecCore* aec) { 43199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Measures the energy in each filter partition and returns the partition with 43299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // highest energy. 43399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // TODO(bjornv): Spread computational cost by computing one partition per 43499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // block? 43599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float wfEnMax = 0; 43699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int i; 43799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int delay = 0; 43899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 43999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (i = 0; i < aec->num_partitions; i++) { 44099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int j; 44199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int pos = i * PART_LEN1; 44299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float wfEn = 0; 44399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_wfEn = _mm_set1_ps(0.0f); 44499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // vectorized code (four at once) 44599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (j = 0; j + 3 < PART_LEN1; j += 4) { 44699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_wfBuf0 = _mm_loadu_ps(&aec->wfBuf[0][pos + j]); 44799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_wfBuf1 = _mm_loadu_ps(&aec->wfBuf[1][pos + j]); 44899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_wfEn = _mm_add_ps(vec_wfEn, _mm_mul_ps(vec_wfBuf0, vec_wfBuf0)); 44999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_wfEn = _mm_add_ps(vec_wfEn, _mm_mul_ps(vec_wfBuf1, vec_wfBuf1)); 45099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 45199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_add_ps_4x1(vec_wfEn, &wfEn); 45299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 45399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // scalar code for the remaining items. 45499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (; j < PART_LEN1; j++) { 45599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org wfEn += aec->wfBuf[0][pos + j] * aec->wfBuf[0][pos + j] + 45699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->wfBuf[1][pos + j] * aec->wfBuf[1][pos + j]; 45799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 45899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 45999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org if (wfEn > wfEnMax) { 46099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org wfEnMax = wfEn; 46199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org delay = i; 46299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 46399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 46499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org return delay; 46599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 46699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 46799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// Updates the following smoothed Power Spectral Densities (PSD): 46899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// - sd : near-end 46999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// - se : residual echo 47099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// - sx : far-end 47199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// - sde : cross-PSD of near-end and residual echo 47299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// - sxd : cross-PSD of near-end and far-end 47399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// 47499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// In addition to updating the PSDs, also the filter diverge state is determined 47599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// upon actions are taken. 47699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.orgstatic void SmoothedPSD(AecCore* aec, 47799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float efw[2][PART_LEN1], 47899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float dfw[2][PART_LEN1], 47999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float xfw[2][PART_LEN1]) { 48099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Power estimate smoothing coefficients. 48199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const float* ptrGCoh = aec->extended_filter_enabled 48299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ? WebRtcAec_kExtendedSmoothingCoefficients[aec->mult - 1] 48399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org : WebRtcAec_kNormalSmoothingCoefficients[aec->mult - 1]; 48499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int i; 48599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float sdSum = 0, seSum = 0; 48699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_15 = _mm_set1_ps(WebRtcAec_kMinFarendPSD); 48799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_GCoh0 = _mm_set1_ps(ptrGCoh[0]); 48899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_GCoh1 = _mm_set1_ps(ptrGCoh[1]); 48999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_sdSum = _mm_set1_ps(0.0f); 49099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_seSum = _mm_set1_ps(0.0f); 49199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 49299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (i = 0; i + 3 < PART_LEN1; i += 4) { 49399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_dfw0 = _mm_loadu_ps(&dfw[0][i]); 49499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_dfw1 = _mm_loadu_ps(&dfw[1][i]); 49599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_efw0 = _mm_loadu_ps(&efw[0][i]); 49699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_efw1 = _mm_loadu_ps(&efw[1][i]); 49799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_xfw0 = _mm_loadu_ps(&xfw[0][i]); 49899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_xfw1 = _mm_loadu_ps(&xfw[1][i]); 49999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_sd = _mm_mul_ps(_mm_loadu_ps(&aec->sd[i]), vec_GCoh0); 50099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_se = _mm_mul_ps(_mm_loadu_ps(&aec->se[i]), vec_GCoh0); 50199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_sx = _mm_mul_ps(_mm_loadu_ps(&aec->sx[i]), vec_GCoh0); 50299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_dfw_sumsq = _mm_mul_ps(vec_dfw0, vec_dfw0); 50399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_efw_sumsq = _mm_mul_ps(vec_efw0, vec_efw0); 50499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_xfw_sumsq = _mm_mul_ps(vec_xfw0, vec_xfw0); 50599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_dfw_sumsq = _mm_add_ps(vec_dfw_sumsq, _mm_mul_ps(vec_dfw1, vec_dfw1)); 50699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_efw_sumsq = _mm_add_ps(vec_efw_sumsq, _mm_mul_ps(vec_efw1, vec_efw1)); 50799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_xfw_sumsq = _mm_add_ps(vec_xfw_sumsq, _mm_mul_ps(vec_xfw1, vec_xfw1)); 50899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_xfw_sumsq = _mm_max_ps(vec_xfw_sumsq, vec_15); 50999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_sd = _mm_add_ps(vec_sd, _mm_mul_ps(vec_dfw_sumsq, vec_GCoh1)); 51099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_se = _mm_add_ps(vec_se, _mm_mul_ps(vec_efw_sumsq, vec_GCoh1)); 51199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_sx = _mm_add_ps(vec_sx, _mm_mul_ps(vec_xfw_sumsq, vec_GCoh1)); 51299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sd[i], vec_sd); 51399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->se[i], vec_se); 51499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sx[i], vec_sx); 51599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 51699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org { 51799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_3210 = _mm_loadu_ps(&aec->sde[i][0]); 51899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_7654 = _mm_loadu_ps(&aec->sde[i + 2][0]); 51999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_a = _mm_shuffle_ps(vec_3210, vec_7654, 52099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(2, 0, 2, 0)); 52199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_b = _mm_shuffle_ps(vec_3210, vec_7654, 52299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(3, 1, 3, 1)); 52399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_dfwefw0011 = _mm_mul_ps(vec_dfw0, vec_efw0); 52499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_dfwefw0110 = _mm_mul_ps(vec_dfw0, vec_efw1); 52599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_a = _mm_mul_ps(vec_a, vec_GCoh0); 52699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_b = _mm_mul_ps(vec_b, vec_GCoh0); 52799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_dfwefw0011 = _mm_add_ps(vec_dfwefw0011, 52899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_dfw1, vec_efw1)); 52999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_dfwefw0110 = _mm_sub_ps(vec_dfwefw0110, 53099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_dfw1, vec_efw0)); 53199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_a = _mm_add_ps(vec_a, _mm_mul_ps(vec_dfwefw0011, vec_GCoh1)); 53299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_b = _mm_add_ps(vec_b, _mm_mul_ps(vec_dfwefw0110, vec_GCoh1)); 53399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sde[i][0], _mm_unpacklo_ps(vec_a, vec_b)); 53499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sde[i + 2][0], _mm_unpackhi_ps(vec_a, vec_b)); 53599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 53699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 53799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org { 53899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_3210 = _mm_loadu_ps(&aec->sxd[i][0]); 53999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_7654 = _mm_loadu_ps(&aec->sxd[i + 2][0]); 54099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_a = _mm_shuffle_ps(vec_3210, vec_7654, 54199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(2, 0, 2, 0)); 54299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_b = _mm_shuffle_ps(vec_3210, vec_7654, 54399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(3, 1, 3, 1)); 54499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_dfwxfw0011 = _mm_mul_ps(vec_dfw0, vec_xfw0); 54599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_dfwxfw0110 = _mm_mul_ps(vec_dfw0, vec_xfw1); 54699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_a = _mm_mul_ps(vec_a, vec_GCoh0); 54799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_b = _mm_mul_ps(vec_b, vec_GCoh0); 54899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_dfwxfw0011 = _mm_add_ps(vec_dfwxfw0011, 54999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_dfw1, vec_xfw1)); 55099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_dfwxfw0110 = _mm_sub_ps(vec_dfwxfw0110, 55199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_dfw1, vec_xfw0)); 55299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_a = _mm_add_ps(vec_a, _mm_mul_ps(vec_dfwxfw0011, vec_GCoh1)); 55399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_b = _mm_add_ps(vec_b, _mm_mul_ps(vec_dfwxfw0110, vec_GCoh1)); 55499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sxd[i][0], _mm_unpacklo_ps(vec_a, vec_b)); 55599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&aec->sxd[i + 2][0], _mm_unpackhi_ps(vec_a, vec_b)); 55699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 55799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 55899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_sdSum = _mm_add_ps(vec_sdSum, vec_sd); 55999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_seSum = _mm_add_ps(vec_seSum, vec_se); 56099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 56199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 56299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_add_ps_4x1(vec_sdSum, &sdSum); 56399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_add_ps_4x1(vec_seSum, &seSum); 56499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 56599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (; i < PART_LEN1; i++) { 56699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sd[i] = ptrGCoh[0] * aec->sd[i] + 56799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (dfw[0][i] * dfw[0][i] + dfw[1][i] * dfw[1][i]); 56899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->se[i] = ptrGCoh[0] * aec->se[i] + 56999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (efw[0][i] * efw[0][i] + efw[1][i] * efw[1][i]); 57099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // We threshold here to protect against the ill-effects of a zero farend. 57199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // The threshold is not arbitrarily chosen, but balances protection and 57299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // adverse interaction with the algorithm's tuning. 57399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // TODO(bjornv): investigate further why this is so sensitive. 57499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sx[i] = 57599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[0] * aec->sx[i] + 57699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * WEBRTC_SPL_MAX( 57799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org xfw[0][i] * xfw[0][i] + xfw[1][i] * xfw[1][i], 57899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org WebRtcAec_kMinFarendPSD); 57999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 58099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sde[i][0] = 58199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[0] * aec->sde[i][0] + 58299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (dfw[0][i] * efw[0][i] + dfw[1][i] * efw[1][i]); 58399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sde[i][1] = 58499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[0] * aec->sde[i][1] + 58599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (dfw[0][i] * efw[1][i] - dfw[1][i] * efw[0][i]); 58699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 58799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sxd[i][0] = 58899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[0] * aec->sxd[i][0] + 58999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (dfw[0][i] * xfw[0][i] + dfw[1][i] * xfw[1][i]); 59099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->sxd[i][1] = 59199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[0] * aec->sxd[i][1] + 59299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org ptrGCoh[1] * (dfw[0][i] * xfw[1][i] - dfw[1][i] * xfw[0][i]); 59399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 59499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org sdSum += aec->sd[i]; 59599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org seSum += aec->se[i]; 59699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 59799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 59899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Divergent filter safeguard. 59999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->divergeState = (aec->divergeState ? 1.05f : 1.0f) * seSum > sdSum; 60099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 60199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org if (aec->divergeState) 60299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org memcpy(efw, dfw, sizeof(efw[0][0]) * 2 * PART_LEN1); 60399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 60499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Reset if error is significantly larger than nearend (13 dB). 60599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org if (!aec->extended_filter_enabled && seSum > (19.95f * sdSum)) 60699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org memset(aec->wfBuf, 0, sizeof(aec->wfBuf)); 60799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 60899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 60999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// Window time domain data to be used by the fft. 61099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org__inline static void WindowData(float* x_windowed, const float* x) { 61199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int i; 61299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (i = 0; i < PART_LEN; i += 4) { 61399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_Buf1 = _mm_loadu_ps(&x[i]); 61499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_Buf2 = _mm_loadu_ps(&x[PART_LEN + i]); 61599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sqrtHanning = _mm_load_ps(&WebRtcAec_sqrtHanning[i]); 61699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // A B C D 61799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_sqrtHanning_rev = 61899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_loadu_ps(&WebRtcAec_sqrtHanning[PART_LEN - i - 3]); 61999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // D C B A 62099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_sqrtHanning_rev = 62199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_shuffle_ps(vec_sqrtHanning_rev, vec_sqrtHanning_rev, 62299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(0, 1, 2, 3)); 62399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&x_windowed[i], _mm_mul_ps(vec_Buf1, vec_sqrtHanning)); 62499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&x_windowed[PART_LEN + i], 62599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_Buf2, vec_sqrtHanning_rev)); 62699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 62799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 62899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 62999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org// Puts fft output data into a complex valued array. 63099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org__inline static void StoreAsComplex(const float* data, 63199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float data_complex[2][PART_LEN1]) { 63299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int i; 63399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (i = 0; i < PART_LEN; i += 4) { 63499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_fft0 = _mm_loadu_ps(&data[2 * i]); 63599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_fft4 = _mm_loadu_ps(&data[2 * i + 4]); 63699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_a = _mm_shuffle_ps(vec_fft0, vec_fft4, 63799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(2, 0, 2, 0)); 63899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_b = _mm_shuffle_ps(vec_fft0, vec_fft4, 63999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(3, 1, 3, 1)); 64099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&data_complex[0][i], vec_a); 64199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&data_complex[1][i], vec_b); 64299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 64399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // fix beginning/end values 64499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org data_complex[1][0] = 0; 64599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org data_complex[1][PART_LEN] = 0; 64699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org data_complex[0][0] = data[0]; 64799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org data_complex[0][PART_LEN] = data[1]; 64899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 64999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 65099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.orgstatic void SubbandCoherenceSSE2(AecCore* aec, 65199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float efw[2][PART_LEN1], 65299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float xfw[2][PART_LEN1], 65399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float* fft, 65499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float* cohde, 65599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float* cohxd) { 65699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org float dfw[2][PART_LEN1]; 65799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org int i; 65899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 65999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org if (aec->delayEstCtr == 0) 66099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->delayIdx = PartitionDelay(aec); 66199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 66299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Use delayed far. 66399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org memcpy(xfw, 66499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec->xfwBuf + aec->delayIdx * PART_LEN1, 66599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org sizeof(xfw[0][0]) * 2 * PART_LEN1); 66699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 66799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Windowed near fft 66899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org WindowData(fft, aec->dBuf); 66999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec_rdft_forward_128(fft); 67099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org StoreAsComplex(fft, dfw); 67199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 67299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Windowed error fft 67399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org WindowData(fft, aec->eBuf); 67499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org aec_rdft_forward_128(fft); 67599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org StoreAsComplex(fft, efw); 67699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 67799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org SmoothedPSD(aec, efw, dfw, xfw); 67899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 67999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org { 68099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_1eminus10 = _mm_set1_ps(1e-10f); 68199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 68299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // Subband coherence 68399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (i = 0; i + 3 < PART_LEN1; i += 4) { 68499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sd = _mm_loadu_ps(&aec->sd[i]); 68599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_se = _mm_loadu_ps(&aec->se[i]); 68699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sx = _mm_loadu_ps(&aec->sx[i]); 68799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sdse = _mm_add_ps(vec_1eminus10, 68899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_sd, vec_se)); 68999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sdsx = _mm_add_ps(vec_1eminus10, 69099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_mul_ps(vec_sd, vec_sx)); 69199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sde_3210 = _mm_loadu_ps(&aec->sde[i][0]); 69299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sde_7654 = _mm_loadu_ps(&aec->sde[i + 2][0]); 69399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sxd_3210 = _mm_loadu_ps(&aec->sxd[i][0]); 69499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sxd_7654 = _mm_loadu_ps(&aec->sxd[i + 2][0]); 69599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sde_0 = _mm_shuffle_ps(vec_sde_3210, vec_sde_7654, 69699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(2, 0, 2, 0)); 69799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sde_1 = _mm_shuffle_ps(vec_sde_3210, vec_sde_7654, 69899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(3, 1, 3, 1)); 69999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sxd_0 = _mm_shuffle_ps(vec_sxd_3210, vec_sxd_7654, 70099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(2, 0, 2, 0)); 70199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org const __m128 vec_sxd_1 = _mm_shuffle_ps(vec_sxd_3210, vec_sxd_7654, 70299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _MM_SHUFFLE(3, 1, 3, 1)); 70399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_cohde = _mm_mul_ps(vec_sde_0, vec_sde_0); 70499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org __m128 vec_cohxd = _mm_mul_ps(vec_sxd_0, vec_sxd_0); 70599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_cohde = _mm_add_ps(vec_cohde, _mm_mul_ps(vec_sde_1, vec_sde_1)); 70699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_cohde = _mm_div_ps(vec_cohde, vec_sdse); 70799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_cohxd = _mm_add_ps(vec_cohxd, _mm_mul_ps(vec_sxd_1, vec_sxd_1)); 70899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org vec_cohxd = _mm_div_ps(vec_cohxd, vec_sdsx); 70999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&cohde[i], vec_cohde); 71099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org _mm_storeu_ps(&cohxd[i], vec_cohxd); 71199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 71299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 71399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org // scalar code for the remaining items. 71499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org for (; i < PART_LEN1; i++) { 71599237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org cohde[i] = 71699237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org (aec->sde[i][0] * aec->sde[i][0] + aec->sde[i][1] * aec->sde[i][1]) / 71799237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org (aec->sd[i] * aec->se[i] + 1e-10f); 71899237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org cohxd[i] = 71999237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org (aec->sxd[i][0] * aec->sxd[i][0] + aec->sxd[i][1] * aec->sxd[i][1]) / 72099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org (aec->sx[i] * aec->sd[i] + 1e-10f); 72199237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 72299237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org } 72399237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org} 72499237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org 725b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.orgvoid WebRtcAec_InitAec_SSE2(void) { 726b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org WebRtcAec_FilterFar = FilterFarSSE2; 727b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org WebRtcAec_ScaleErrorSignal = ScaleErrorSignalSSE2; 728b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org WebRtcAec_FilterAdaptation = FilterAdaptationSSE2; 729b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org WebRtcAec_OverdriveAndSuppress = OverdriveAndSuppressSSE2; 73099237f424cf6c3bd76488e4a50b10f0f3c53a5d3bjornv@webrtc.org WebRtcAec_SubbandCoherence = SubbandCoherenceSSE2; 731b015cbede88899f67a53fbbe581b02ce8e32794andrew@webrtc.org} 732