1e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent/* 2e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. 3e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * 4e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * Use of this source code is governed by a BSD-style license 5e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * that can be found in the LICENSE file in the root of the source 6e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * tree. An additional intellectual property rights grant can be found 7e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * in the file PATENTS. All contributing project authors may 8e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * be found in the AUTHORS file in the root of the source tree. 9e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent */ 10e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 11e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent/* 12e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent * The core AEC algorithm, SSE2 version of speed-critical functions. 13e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent */ 14e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 15c55a96383497a772a307b346368133960b02ad03Eric Laurent#include "typedefs.h" 16c55a96383497a772a307b346368133960b02ad03Eric Laurent 17c55a96383497a772a307b346368133960b02ad03Eric Laurent#if defined(WEBRTC_USE_SSE2) 18e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent#include <emmintrin.h> 19e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent#include <math.h> 20e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 21e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent#include "aec_core.h" 22e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent#include "aec_rdft.h" 23e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 24e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent__inline static float MulRe(float aRe, float aIm, float bRe, float bIm) 25e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent{ 26e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent return aRe * bRe - aIm * bIm; 27e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 28e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 29e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent__inline static float MulIm(float aRe, float aIm, float bRe, float bIm) 30e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent{ 31e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent return aRe * bIm + aIm * bRe; 32e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 33e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 34e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentstatic void FilterFarSSE2(aec_t *aec, float yf[2][PART_LEN1]) 35e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent{ 36e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int i; 37e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (i = 0; i < NR_PART; i++) { 38e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int j; 39e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int xPos = (i + aec->xfBufBlockPos) * PART_LEN1; 40e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int pos = i * PART_LEN1; 41e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Check for wrap 42e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent if (i + aec->xfBufBlockPos >= NR_PART) { 43e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent xPos -= NR_PART*(PART_LEN1); 44e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 45e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 46e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // vectorized code (four at once) 47e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (j = 0; j + 3 < PART_LEN1; j += 4) { 48e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xfBuf_re = _mm_loadu_ps(&aec->xfBuf[0][xPos + j]); 49e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xfBuf_im = _mm_loadu_ps(&aec->xfBuf[1][xPos + j]); 50e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 wfBuf_re = _mm_loadu_ps(&aec->wfBuf[0][pos + j]); 51e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 wfBuf_im = _mm_loadu_ps(&aec->wfBuf[1][pos + j]); 52e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 yf_re = _mm_loadu_ps(&yf[0][j]); 53e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 yf_im = _mm_loadu_ps(&yf[1][j]); 54e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 a = _mm_mul_ps(xfBuf_re, wfBuf_re); 55e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 b = _mm_mul_ps(xfBuf_im, wfBuf_im); 56e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 c = _mm_mul_ps(xfBuf_re, wfBuf_im); 57e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 d = _mm_mul_ps(xfBuf_im, wfBuf_re); 58e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 e = _mm_sub_ps(a, b); 59e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 f = _mm_add_ps(c, d); 60e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 g = _mm_add_ps(yf_re, e); 61e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 h = _mm_add_ps(yf_im, f); 62e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&yf[0][j], g); 63e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&yf[1][j], h); 64e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 65e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // scalar code for the remaining items. 66e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (; j < PART_LEN1; j++) { 67e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent yf[0][j] += MulRe(aec->xfBuf[0][xPos + j], aec->xfBuf[1][xPos + j], 68e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec->wfBuf[0][ pos + j], aec->wfBuf[1][ pos + j]); 69e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent yf[1][j] += MulIm(aec->xfBuf[0][xPos + j], aec->xfBuf[1][xPos + j], 70e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec->wfBuf[0][ pos + j], aec->wfBuf[1][ pos + j]); 71e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 72e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 73e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 74e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 75e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentstatic void ScaleErrorSignalSSE2(aec_t *aec, float ef[2][PART_LEN1]) 76e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent{ 77e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 k1e_10f = _mm_set1_ps(1e-10f); 78e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 kThresh = _mm_set1_ps(aec->errThresh); 79e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 kMu = _mm_set1_ps(aec->mu); 80e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 81e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int i; 82e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // vectorized code (four at once) 83e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (i = 0; i + 3 < PART_LEN1; i += 4) { 84e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xPow = _mm_loadu_ps(&aec->xPow[i]); 85e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_re_base = _mm_loadu_ps(&ef[0][i]); 86e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_im_base = _mm_loadu_ps(&ef[1][i]); 87e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 88e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xPowPlus = _mm_add_ps(xPow, k1e_10f); 89e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 ef_re = _mm_div_ps(ef_re_base, xPowPlus); 90e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 ef_im = _mm_div_ps(ef_im_base, xPowPlus); 91e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_re2 = _mm_mul_ps(ef_re, ef_re); 92e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_im2 = _mm_mul_ps(ef_im, ef_im); 93e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_sum2 = _mm_add_ps(ef_re2, ef_im2); 94e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 absEf = _mm_sqrt_ps(ef_sum2); 95e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 bigger = _mm_cmpgt_ps(absEf, kThresh); 96e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 absEfPlus = _mm_add_ps(absEf, k1e_10f); 97e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 absEfInv = _mm_div_ps(kThresh, absEfPlus); 98e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 ef_re_if = _mm_mul_ps(ef_re, absEfInv); 99e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 ef_im_if = _mm_mul_ps(ef_im, absEfInv); 100e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_re_if = _mm_and_ps(bigger, ef_re_if); 101e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_im_if = _mm_and_ps(bigger, ef_im_if); 102e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_re = _mm_andnot_ps(bigger, ef_re); 103e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_im = _mm_andnot_ps(bigger, ef_im); 104e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_re = _mm_or_ps(ef_re, ef_re_if); 105e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_im = _mm_or_ps(ef_im, ef_im_if); 106e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_re = _mm_mul_ps(ef_re, kMu); 107e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef_im = _mm_mul_ps(ef_im, kMu); 108e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 109e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&ef[0][i], ef_re); 110e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&ef[1][i], ef_im); 111e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 112e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // scalar code for the remaining items. 113e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (; i < (PART_LEN1); i++) { 114e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent float absEf; 115e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[0][i] /= (aec->xPow[i] + 1e-10f); 116e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[1][i] /= (aec->xPow[i] + 1e-10f); 117e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent absEf = sqrtf(ef[0][i] * ef[0][i] + ef[1][i] * ef[1][i]); 118e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 119e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent if (absEf > aec->errThresh) { 120e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent absEf = aec->errThresh / (absEf + 1e-10f); 121e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[0][i] *= absEf; 122e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[1][i] *= absEf; 123e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 124e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 125e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Stepsize factor 126e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[0][i] *= aec->mu; 127e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[1][i] *= aec->mu; 128e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 129e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 130e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 131e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentstatic void FilterAdaptationSSE2(aec_t *aec, float *fft, float ef[2][PART_LEN1]) { 132e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int i, j; 133e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (i = 0; i < NR_PART; i++) { 134e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int xPos = (i + aec->xfBufBlockPos)*(PART_LEN1); 135e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int pos = i * PART_LEN1; 136e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Check for wrap 137e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent if (i + aec->xfBufBlockPos >= NR_PART) { 138e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent xPos -= NR_PART * PART_LEN1; 139e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 140e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 141e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Process the whole array... 142e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (j = 0; j < PART_LEN; j+= 4) { 143e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Load xfBuf and ef. 144e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xfBuf_re = _mm_loadu_ps(&aec->xfBuf[0][xPos + j]); 145e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 xfBuf_im = _mm_loadu_ps(&aec->xfBuf[1][xPos + j]); 146e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_re = _mm_loadu_ps(&ef[0][j]); 147e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 ef_im = _mm_loadu_ps(&ef[1][j]); 148e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Calculate the product of conjugate(xfBuf) by ef. 149e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // re(conjugate(a) * b) = aRe * bRe + aIm * bIm 150e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // im(conjugate(a) * b)= aRe * bIm - aIm * bRe 151e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 a = _mm_mul_ps(xfBuf_re, ef_re); 152e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 b = _mm_mul_ps(xfBuf_im, ef_im); 153e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 c = _mm_mul_ps(xfBuf_re, ef_im); 154e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 d = _mm_mul_ps(xfBuf_im, ef_re); 155e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 e = _mm_add_ps(a, b); 156e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 f = _mm_sub_ps(c, d); 157e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Interleave real and imaginary parts. 158e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 g = _mm_unpacklo_ps(e, f); 159e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 h = _mm_unpackhi_ps(e, f); 160e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Store 161e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&fft[2*j + 0], g); 162e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&fft[2*j + 4], h); 163e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 164e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // ... and fixup the first imaginary entry. 165e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent fft[1] = MulRe(aec->xfBuf[0][xPos + PART_LEN], 166e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent -aec->xfBuf[1][xPos + PART_LEN], 167e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent ef[0][PART_LEN], ef[1][PART_LEN]); 168e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 169e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec_rdft_inverse_128(fft); 170e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent memset(fft + PART_LEN, 0, sizeof(float)*PART_LEN); 171e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 172e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // fft scaling 173e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 174e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent float scale = 2.0f / PART_LEN2; 175e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 scale_ps = _mm_load_ps1(&scale); 176e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (j = 0; j < PART_LEN; j+=4) { 177e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft_ps = _mm_loadu_ps(&fft[j]); 178e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft_scale = _mm_mul_ps(fft_ps, scale_ps); 179e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&fft[j], fft_scale); 180e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 181e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 182e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec_rdft_forward_128(fft); 183e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 184e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 185e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent float wt1 = aec->wfBuf[1][pos]; 186e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec->wfBuf[0][pos + PART_LEN] += fft[1]; 187e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (j = 0; j < PART_LEN; j+= 4) { 188e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 wtBuf_re = _mm_loadu_ps(&aec->wfBuf[0][pos + j]); 189e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 wtBuf_im = _mm_loadu_ps(&aec->wfBuf[1][pos + j]); 190e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft0 = _mm_loadu_ps(&fft[2 * j + 0]); 191e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft4 = _mm_loadu_ps(&fft[2 * j + 4]); 192e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft_re = _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(2, 0, 2 ,0)); 193e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 fft_im = _mm_shuffle_ps(fft0, fft4, _MM_SHUFFLE(3, 1, 3 ,1)); 194e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent wtBuf_re = _mm_add_ps(wtBuf_re, fft_re); 195e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent wtBuf_im = _mm_add_ps(wtBuf_im, fft_im); 196e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&aec->wfBuf[0][pos + j], wtBuf_re); 197e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&aec->wfBuf[1][pos + j], wtBuf_im); 198e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 199e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent aec->wfBuf[1][pos] = wt1; 200e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 201e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 202e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 203e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 204e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentstatic __m128 mm_pow_ps(__m128 a, __m128 b) 205e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent{ 206e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // a^b = exp2(b * log2(a)) 207e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // exp2(x) and log2(x) are calculated using polynomial approximations. 208e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 log2_a, b_log2_a, a_exp_b; 209e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 210e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Calculate log2(x), x = a. 211e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 212e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // To calculate log2(x), we decompose x like this: 213e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // x = y * 2^n 214e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // n is an integer 215e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // y is in the [1.0, 2.0) range 216e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // 217e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // log2(x) = log2(y) + n 218e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // n can be evaluated by playing with float representation. 219e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // log2(y) in a small range can be approximated, this code uses an order 220e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // five polynomial approximation. The coefficients have been 221e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // estimated with the Remez algorithm and the resulting 222e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // polynomial has a maximum relative error of 0.00086%. 223e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 224e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Compute n. 225e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // This is done by masking the exponent, shifting it into the top bit of 226e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // the mantissa, putting eight into the biased exponent (to shift/ 227e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // compensate the fact that the exponent has been shifted in the top/ 228e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // fractional part and finally getting rid of the implicit leading one 229e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // from the mantissa by substracting it out. 230e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int float_exponent_mask[4] ALIGN16_END = 231e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0x7F800000, 0x7F800000, 0x7F800000, 0x7F800000}; 232e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int eight_biased_exponent[4] ALIGN16_END = 233e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0x43800000, 0x43800000, 0x43800000, 0x43800000}; 234e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int implicit_leading_one[4] ALIGN16_END = 235e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0x43BF8000, 0x43BF8000, 0x43BF8000, 0x43BF8000}; 236e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const int shift_exponent_into_top_mantissa = 8; 237e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 two_n = _mm_and_ps(a, *((__m128 *)float_exponent_mask)); 238c55a96383497a772a307b346368133960b02ad03Eric Laurent const __m128 n_1 = _mm_castsi128_ps(_mm_srli_epi32(_mm_castps_si128(two_n), 239c55a96383497a772a307b346368133960b02ad03Eric Laurent shift_exponent_into_top_mantissa)); 240c55a96383497a772a307b346368133960b02ad03Eric Laurent const __m128 n_0 = _mm_or_ps(n_1, *((__m128 *)eight_biased_exponent)); 241e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 n = _mm_sub_ps(n_0, *((__m128 *)implicit_leading_one)); 242e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 243e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Compute y. 244e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int mantissa_mask[4] ALIGN16_END = 245e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0x007FFFFF, 0x007FFFFF, 0x007FFFFF, 0x007FFFFF}; 246e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int zero_biased_exponent_is_one[4] ALIGN16_END = 247e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0x3F800000, 0x3F800000, 0x3F800000, 0x3F800000}; 248e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 mantissa = _mm_and_ps(a, *((__m128 *)mantissa_mask)); 249e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 y = _mm_or_ps( 250e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent mantissa, *((__m128 *)zero_biased_exponent_is_one)); 251e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 252e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Approximate log2(y) ~= (y - 1) * pol5(y). 253e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // pol5(y) = C5 * y^5 + C4 * y^4 + C3 * y^3 + C2 * y^2 + C1 * y + C0 254e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C5[4] = 255e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {-3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f, -3.4436006e-2f}; 256e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C4[4] = 257e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f, 3.1821337e-1f}; 258e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C3[4] = 259e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {-1.2315303f, -1.2315303f, -1.2315303f, -1.2315303f}; 260e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C2[4] = 261e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {2.5988452f, 2.5988452f, 2.5988452f, 2.5988452f}; 262e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C1[4] = 263e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {-3.3241990f, -3.3241990f, -3.3241990f, -3.3241990f}; 264e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float ALIGN16_END C0[4] = 265e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {3.1157899f, 3.1157899f, 3.1157899f, 3.1157899f}; 266e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_0 = _mm_mul_ps(y, *((__m128 *)C5)); 267e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_1 = _mm_add_ps(pol5_y_0, *((__m128 *)C4)); 268e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_2 = _mm_mul_ps(pol5_y_1, y); 269e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_3 = _mm_add_ps(pol5_y_2, *((__m128 *)C3)); 270e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_4 = _mm_mul_ps(pol5_y_3, y); 271e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_5 = _mm_add_ps(pol5_y_4, *((__m128 *)C2)); 272e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_6 = _mm_mul_ps(pol5_y_5, y); 273e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_7 = _mm_add_ps(pol5_y_6, *((__m128 *)C1)); 274e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y_8 = _mm_mul_ps(pol5_y_7, y); 275e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 pol5_y = _mm_add_ps(pol5_y_8, *((__m128 *)C0)); 276e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 y_minus_one = _mm_sub_ps( 277e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent y, *((__m128 *)zero_biased_exponent_is_one)); 278e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 log2_y = _mm_mul_ps(y_minus_one , pol5_y); 279e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 280e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Combine parts. 281e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent log2_a = _mm_add_ps(n, log2_y); 282e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 283e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 284e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // b * log2(a) 285e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent b_log2_a = _mm_mul_ps(b, log2_a); 286e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 287e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Calculate exp2(x), x = b * log2(a). 288e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 289e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // To calculate 2^x, we decompose x like this: 290e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // x = n + y 291e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // n is an integer, the value of x - 0.5 rounded down, therefore 292e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // y is in the [0.5, 1.5) range 293e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // 294e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // 2^x = 2^n * 2^y 295e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // 2^n can be evaluated by playing with float representation. 296e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // 2^y in a small range can be approximated, this code uses an order two 297e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // polynomial approximation. The coefficients have been estimated 298e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // with the Remez algorithm and the resulting polynomial has a 299e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // maximum relative error of 0.17%. 300e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 301e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // To avoid over/underflow, we reduce the range of input to ]-127, 129]. 302e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float max_input[4] ALIGN16_END = 303e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {129.f, 129.f, 129.f, 129.f}; 304e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float min_input[4] ALIGN16_END = 305e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {-126.99999f, -126.99999f, -126.99999f, -126.99999f}; 306e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 x_min = _mm_min_ps(b_log2_a, *((__m128 *)max_input)); 307e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 x_max = _mm_max_ps(x_min, *((__m128 *)min_input)); 308e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Compute n. 309e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float half[4] ALIGN16_END = 310e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {0.5f, 0.5f, 0.5f, 0.5f}; 311e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 x_minus_half = _mm_sub_ps(x_max, *((__m128 *)half)); 312e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128i x_minus_half_floor = _mm_cvtps_epi32(x_minus_half); 313e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Compute 2^n. 314e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG int float_exponent_bias[4] ALIGN16_END = 315e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {127, 127, 127, 127}; 316e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const int float_exponent_shift = 23; 317e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128i two_n_exponent = _mm_add_epi32( 318e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent x_minus_half_floor, *((__m128i *)float_exponent_bias)); 319c55a96383497a772a307b346368133960b02ad03Eric Laurent const __m128 two_n = _mm_castsi128_ps(_mm_slli_epi32( 320c55a96383497a772a307b346368133960b02ad03Eric Laurent two_n_exponent, float_exponent_shift)); 321e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Compute y. 322e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 y = _mm_sub_ps(x_max, _mm_cvtepi32_ps(x_minus_half_floor)); 323e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Approximate 2^y ~= C2 * y^2 + C1 * y + C0. 324e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float C2[4] ALIGN16_END = 325e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f, 3.3718944e-1f}; 326e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float C1[4] ALIGN16_END = 327e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f, 6.5763628e-1f}; 328e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent static const ALIGN16_BEG float C0[4] ALIGN16_END = 329e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent {1.0017247f, 1.0017247f, 1.0017247f, 1.0017247f}; 330e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 exp2_y_0 = _mm_mul_ps(y, *((__m128 *)C2)); 331e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 exp2_y_1 = _mm_add_ps(exp2_y_0, *((__m128 *)C1)); 332e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 exp2_y_2 = _mm_mul_ps(exp2_y_1, y); 333e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 exp2_y = _mm_add_ps(exp2_y_2, *((__m128 *)C0)); 334e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 335e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Combine parts. 336e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent a_exp_b = _mm_mul_ps(exp2_y, two_n); 337e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 338e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent return a_exp_b; 339e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 340e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 341e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentextern const float WebRtcAec_weightCurve[65]; 342e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentextern const float WebRtcAec_overDriveCurve[65]; 343e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 344e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentstatic void OverdriveAndSuppressSSE2(aec_t *aec, float hNl[PART_LEN1], 345e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const float hNlFb, 346e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent float efw[2][PART_LEN1]) { 347e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent int i; 348e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_hNlFb = _mm_set1_ps(hNlFb); 349e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_one = _mm_set1_ps(1.0f); 350e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_minus_one = _mm_set1_ps(-1.0f); 351e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_overDriveSm = _mm_set1_ps(aec->overDriveSm); 352e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // vectorized code (four at once) 353e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (i = 0; i + 3 < PART_LEN1; i+=4) { 354e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Weight subbands 355e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 vec_hNl = _mm_loadu_ps(&hNl[i]); 356e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_weightCurve = _mm_loadu_ps(&WebRtcAec_weightCurve[i]); 357e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 bigger = _mm_cmpgt_ps(vec_hNl, vec_hNlFb); 358e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_weightCurve_hNlFb = _mm_mul_ps( 359e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_weightCurve, vec_hNlFb); 360e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_one_weightCurve = _mm_sub_ps(vec_one, vec_weightCurve); 361e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_one_weightCurve_hNl = _mm_mul_ps( 362e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_one_weightCurve, vec_hNl); 363e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_if0 = _mm_andnot_ps(bigger, vec_hNl); 364e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_if1 = _mm_and_ps( 365e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent bigger, _mm_add_ps(vec_weightCurve_hNlFb, vec_one_weightCurve_hNl)); 366e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_hNl = _mm_or_ps(vec_if0, vec_if1); 367e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 368e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 369e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_overDriveCurve = _mm_loadu_ps( 370e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent &WebRtcAec_overDriveCurve[i]); 371e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent const __m128 vec_overDriveSm_overDriveCurve = _mm_mul_ps( 372e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_overDriveSm, vec_overDriveCurve); 373e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_hNl = mm_pow_ps(vec_hNl, vec_overDriveSm_overDriveCurve); 374e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&hNl[i], vec_hNl); 375e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 376e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 377e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Suppress error signal 378e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent { 379e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 vec_efw_re = _mm_loadu_ps(&efw[0][i]); 380e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent __m128 vec_efw_im = _mm_loadu_ps(&efw[1][i]); 381e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_efw_re = _mm_mul_ps(vec_efw_re, vec_hNl); 382e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_efw_im = _mm_mul_ps(vec_efw_im, vec_hNl); 383e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 384e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Ooura fft returns incorrect sign on imaginary component. It matters 385e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // here because we are making an additive change with comfort noise. 386e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent vec_efw_im = _mm_mul_ps(vec_efw_im, vec_minus_one); 387e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&efw[0][i], vec_efw_re); 388e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent _mm_storeu_ps(&efw[1][i], vec_efw_im); 389e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 390e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 391e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // scalar code for the remaining items. 392e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent for (; i < PART_LEN1; i++) { 393e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Weight subbands 394e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent if (hNl[i] > hNlFb) { 395e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent hNl[i] = WebRtcAec_weightCurve[i] * hNlFb + 396e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent (1 - WebRtcAec_weightCurve[i]) * hNl[i]; 397e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 398e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent hNl[i] = powf(hNl[i], aec->overDriveSm * WebRtcAec_overDriveCurve[i]); 399e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 400e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Suppress error signal 401e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent efw[0][i] *= hNl[i]; 402e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent efw[1][i] *= hNl[i]; 403e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 404e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // Ooura fft returns incorrect sign on imaginary component. It matters 405e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent // here because we are making an additive change with comfort noise. 406e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent efw[1][i] *= -1; 407e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent } 408e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 409e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 410e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurentvoid WebRtcAec_InitAec_SSE2(void) { 411e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent WebRtcAec_FilterFar = FilterFarSSE2; 412e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent WebRtcAec_ScaleErrorSignal = ScaleErrorSignalSSE2; 413e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent WebRtcAec_FilterAdaptation = FilterAdaptationSSE2; 414e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent WebRtcAec_OverdriveAndSuppress = OverdriveAndSuppressSSE2; 415e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent} 416e48d5845c8b35de2ab73ea055c18a61fa3a9f0beEric Laurent 417c55a96383497a772a307b346368133960b02ad03Eric Laurent#endif // WEBRTC_USE_SSE2 418