10c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org/* 20c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 30c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * 40c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * Use of this source code is governed by a BSD-style license 50c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * that can be found in the LICENSE file in the root of the source 60c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * tree. An additional intellectual property rights grant can be found 70c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * in the file PATENTS. All contributing project authors may 80c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org * be found in the AUTHORS file in the root of the source tree. 90c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org */ 100c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 110c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org#define _USE_MATH_DEFINES 120c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 130f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.org#include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h" 140c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 150c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org#include <algorithm> 160c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org#include <cmath> 17645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald#include <numeric> 180f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.org#include <vector> 190c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 2092a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org#include "webrtc/base/arraysize.h" 210c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org#include "webrtc/common_audio/window_generator.h" 220c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org#include "webrtc/modules/audio_processing/beamformer/covariance_matrix_generator.h" 230c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 240c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.orgnamespace webrtc { 250c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.orgnamespace { 260c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 270c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// Alpha for the Kaiser Bessel Derived window. 28645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst float kKbdAlpha = 1.5f; 290c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 3027669f320b26bdb8cce4e8a0ab1fd6654bde529baluebs@webrtc.orgconst float kSpeedOfSoundMeterSeconds = 343; 310c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 324a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs// The minimum separation in radians between the target direction and an 334a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs// interferer scenario. 344a66e4a4d8847cef1e61aec6978f64dddece3d96aluebsconst float kMinAwayRadians = 0.2f; 354a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs 364a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs// The separation between the target direction and the closest interferer 374a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs// scenario is proportional to this constant. 384a66e4a4d8847cef1e61aec6978f64dddece3d96aluebsconst float kAwaySlope = 0.008f; 394a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs 40ec4521cdb488997ce19bc3ed8ddc4b327187e609aluebs@webrtc.org// When calculating the interference covariance matrix, this is the weight for 410c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// the weighted average between the uniform covariance matrix and the angled 420c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// covariance matrix. 430c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// Rpsi = Rpsi_angled * kBalance + Rpsi_uniform * (1 - kBalance) 4445daf7b26f49793c30e395f7ba7be30aa51936bbaluebsconst float kBalance = 0.95f; 450c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 46645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Alpha coefficients for mask smoothing. 47645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst float kMaskTimeSmoothAlpha = 0.2f; 48645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst float kMaskFrequencySmoothAlpha = 0.6f; 490c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 5092a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org// The average mask is computed from masks in this mid-frequency range. If these 5192a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org// ranges are changed |kMaskQuantile| might need to be adjusted. 52645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst int kLowMeanStartHz = 200; 53645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst int kLowMeanEndHz = 400; 540c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 5545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// Range limiter for subtractive terms in the nominator and denominator of the 5645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// postfilter expression. It handles the scenario mismatch between the true and 5745daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// model sources (target and interference). 5845daf7b26f49793c30e395f7ba7be30aa51936bbaluebsconst float kCutOffConstant = 0.9999f; 5945daf7b26f49793c30e395f7ba7be30aa51936bbaluebs 6092a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org// Quantile of mask values which is used to estimate target presence. 61645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldconst float kMaskQuantile = 0.7f; 62d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org// Mask threshold over which the data is considered signal and not interference. 6345daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// It has to be updated every time the postfilter calculation is changed 6445daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// significantly. 6545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// TODO(aluebs): Write a tool to tune the target threshold automatically based 6645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// on files annotated with target and interference ground truth. 6745daf7b26f49793c30e395f7ba7be30aa51936bbaluebsconst float kMaskTargetThreshold = 0.01f; 68d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org// Time in seconds after which the data is considered interference if the mask 69d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org// does not pass |kMaskTargetThreshold|. 70d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.orgconst float kHoldTargetSeconds = 0.25f; 71d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org 7245daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// To compensate for the attenuation this algorithm introduces to the target 7345daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// signal. It was estimated empirically from a low-noise low-reverberation 7445daf7b26f49793c30e395f7ba7be30aa51936bbaluebs// recording from broadside. 7545daf7b26f49793c30e395f7ba7be30aa51936bbaluebsconst float kCompensationGain = 2.f; 7645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs 770c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// Does conjugate(|norm_mat|) * |mat| * transpose(|norm_mat|). No extra space is 780c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// used; to accomplish this, we compute both multiplications in the same loop. 7991ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org// The returned norm is clamped to be non-negative. 800c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.orgfloat Norm(const ComplexMatrix<float>& mat, 810c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const ComplexMatrix<float>& norm_mat) { 826955870806624479723addfae6dcf5d13968796cPeter Kasting RTC_CHECK_EQ(1u, norm_mat.num_rows()); 8391d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_rows()); 8491d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(norm_mat.num_columns(), mat.num_columns()); 850c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 860c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex<float> first_product = complex<float>(0.f, 0.f); 870c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex<float> second_product = complex<float>(0.f, 0.f); 880c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 890c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const complex<float>* const* mat_els = mat.elements(); 900c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const complex<float>* const* norm_mat_els = norm_mat.elements(); 910c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 926955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t i = 0; i < norm_mat.num_columns(); ++i) { 936955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t j = 0; j < norm_mat.num_columns(); ++j) { 94661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org first_product += conj(norm_mat_els[0][j]) * mat_els[j][i]; 950c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 960c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org second_product += first_product * norm_mat_els[0][i]; 970c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org first_product = 0.f; 980c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 9991ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org return std::max(second_product.real(), 0.f); 1000c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 1010c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1020c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// Does conjugate(|lhs|) * |rhs| for row vectors |lhs| and |rhs|. 1030c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.orgcomplex<float> ConjugateDotProduct(const ComplexMatrix<float>& lhs, 1040c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const ComplexMatrix<float>& rhs) { 1056955870806624479723addfae6dcf5d13968796cPeter Kasting RTC_CHECK_EQ(1u, lhs.num_rows()); 1066955870806624479723addfae6dcf5d13968796cPeter Kasting RTC_CHECK_EQ(1u, rhs.num_rows()); 10791d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(lhs.num_columns(), rhs.num_columns()); 1080c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1090c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const complex<float>* const* lhs_elements = lhs.elements(); 1100c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org const complex<float>* const* rhs_elements = rhs.elements(); 1110c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1120c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex<float> result = complex<float>(0.f, 0.f); 1136955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t i = 0; i < lhs.num_columns(); ++i) { 1140c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org result += conj(lhs_elements[0][i]) * rhs_elements[0][i]; 1150c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 1160c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1170c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org return result; 1180c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 1190c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1200c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org// Works for positive numbers only. 121dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kastingsize_t Round(float x) { 122dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting return static_cast<size_t>(std::floor(x + 0.5f)); 1230c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 1240c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1252a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org// Calculates the sum of absolute values of a complex matrix. 1262a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.orgfloat SumAbs(const ComplexMatrix<float>& mat) { 1272a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org float sum_abs = 0.f; 1282a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org const complex<float>* const* mat_els = mat.elements(); 1296955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t i = 0; i < mat.num_rows(); ++i) { 1306955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t j = 0; j < mat.num_columns(); ++j) { 1312a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org sum_abs += std::abs(mat_els[i][j]); 1322a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org } 1332a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org } 1342a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org return sum_abs; 1352a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org} 1362a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org 137661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org// Calculates the sum of squares of a complex matrix. 138661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.orgfloat SumSquares(const ComplexMatrix<float>& mat) { 139661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org float sum_squares = 0.f; 140661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org const complex<float>* const* mat_els = mat.elements(); 1416955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t i = 0; i < mat.num_rows(); ++i) { 1426955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t j = 0; j < mat.num_columns(); ++j) { 143661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org float abs_value = std::abs(mat_els[i][j]); 144661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org sum_squares += abs_value * abs_value; 145661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org } 146661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org } 147661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org return sum_squares; 148661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org} 149661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org 1501d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org// Does |out| = |in|.' * conj(|in|) for row vector |in|. 1511d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.orgvoid TransposedConjugatedProduct(const ComplexMatrix<float>& in, 1521d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org ComplexMatrix<float>* out) { 1536955870806624479723addfae6dcf5d13968796cPeter Kasting RTC_CHECK_EQ(1u, in.num_rows()); 15491d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(out->num_rows(), in.num_columns()); 15591d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_CHECK_EQ(out->num_columns(), in.num_columns()); 1561d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org const complex<float>* in_elements = in.elements()[0]; 1571d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org complex<float>* const* out_elements = out->elements(); 1586955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t i = 0; i < out->num_rows(); ++i) { 1596955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t j = 0; j < out->num_columns(); ++j) { 1601d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org out_elements[i][j] = in_elements[i] * conj(in_elements[j]); 1611d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org } 1621d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org } 1631d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org} 1641d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org 1651d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.orgstd::vector<Point> GetCenteredArray(std::vector<Point> array_geometry) { 16625702cb1628941427fa55e528f53483f239ae011pkasting for (size_t dim = 0; dim < 3; ++dim) { 1671d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org float center = 0.f; 1681d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org for (size_t i = 0; i < array_geometry.size(); ++i) { 1691d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org center += array_geometry[i].c[dim]; 1701d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org } 1711d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org center /= array_geometry.size(); 1721d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org for (size_t i = 0; i < array_geometry.size(); ++i) { 1731d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org array_geometry[i].c[dim] -= center; 1741d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org } 1751d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org } 1761d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org return array_geometry; 1771d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org} 1781d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org 1790c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} // namespace 1800c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 181cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsconst float NonlinearBeamformer::kHalfBeamWidthRadians = DegreesToRadians(20.f); 182cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 183dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting// static 184dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kastingconst size_t NonlinearBeamformer::kNumFreqBins; 185dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting 1860f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgNonlinearBeamformer::NonlinearBeamformer( 187cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs const std::vector<Point>& array_geometry, 188cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs SphericalPointf target_direction) 1894a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs : num_input_channels_(array_geometry.size()), 1904a66e4a4d8847cef1e61aec6978f64dddece3d96aluebs array_geometry_(GetCenteredArray(array_geometry)), 191cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs array_normal_(GetArrayNormalIfExists(array_geometry)), 192cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs min_mic_spacing_(GetMinimumSpacing(array_geometry)), 193cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs target_angle_radians_(target_direction.azimuth()), 194cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs away_radians_(std::min( 195cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs static_cast<float>(M_PI), 196cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs std::max(kMinAwayRadians, 197cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs kAwaySlope * static_cast<float>(M_PI) / min_mic_spacing_))) { 198645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald WindowGenerator::KaiserBesselDerived(kKbdAlpha, kFftSize, window_); 199d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org} 200d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org 2010f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::Initialize(int chunk_size_ms, int sample_rate_hz) { 202dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting chunk_length_ = 203dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting static_cast<size_t>(sample_rate_hz / (1000.f / chunk_size_ms)); 204d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org sample_rate_hz_ = sample_rate_hz; 205645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald 20692a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org high_pass_postfilter_mask_ = 1.f; 207d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org is_target_present_ = false; 208d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org hold_target_blocks_ = kHoldTargetSeconds * 2 * sample_rate_hz / kFftSize; 209d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org interference_blocks_count_ = hold_target_blocks_; 210d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org 2110c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org lapped_transform_.reset(new LappedTransform(num_input_channels_, 2120c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 1, 2130c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org chunk_length_, 214d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org window_, 2150c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org kFftSize, 2160c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org kFftSize / 2, 2170c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org this)); 218dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = 0; i < kNumFreqBins; ++i) { 219645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald time_smooth_mask_[i] = 1.f; 220645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald final_mask_[i] = 1.f; 2210c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float freq_hz = (static_cast<float>(i) / kFftSize) * sample_rate_hz_; 2220c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org wave_numbers_[i] = 2 * M_PI * freq_hz / kSpeedOfSoundMeterSeconds; 2230c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 2240c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 225cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitLowFrequencyCorrectionRanges(); 226cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitDiffuseCovMats(); 227cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs AimAt(SphericalPointf(target_angle_radians_, 0.f, 1.f)); 228cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs} 2290c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 230cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// These bin indexes determine the regions over which a mean is taken. This is 231cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// applied as a constant value over the adjacent end "frequency correction" 232cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// regions. 233cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// 234cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// low_mean_start_bin_ high_mean_start_bin_ 235cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// v v constant 236cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// |----------------|--------|----------------|-------|----------------| 237cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// constant ^ ^ 238cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// low_mean_end_bin_ high_mean_end_bin_ 239cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs// 240cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsvoid NonlinearBeamformer::InitLowFrequencyCorrectionRanges() { 241cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs low_mean_start_bin_ = Round(kLowMeanStartHz * kFftSize / sample_rate_hz_); 242cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs low_mean_end_bin_ = Round(kLowMeanEndHz * kFftSize / sample_rate_hz_); 243cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 244cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs RTC_DCHECK_GT(low_mean_start_bin_, 0U); 245cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs RTC_DCHECK_LT(low_mean_start_bin_, low_mean_end_bin_); 2460c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 2470c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 248cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsvoid NonlinearBeamformer::InitHighFrequencyCorrectionRanges() { 2491897f77806baad82d321efb6c4c2f27f8c9458b3aluebs const float kAliasingFreqHz = 2501897f77806baad82d321efb6c4c2f27f8c9458b3aluebs kSpeedOfSoundMeterSeconds / 251cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs (min_mic_spacing_ * (1.f + std::abs(std::cos(target_angle_radians_)))); 2521897f77806baad82d321efb6c4c2f27f8c9458b3aluebs const float kHighMeanStartHz = std::min(0.5f * kAliasingFreqHz, 2531897f77806baad82d321efb6c4c2f27f8c9458b3aluebs sample_rate_hz_ / 2.f); 2541897f77806baad82d321efb6c4c2f27f8c9458b3aluebs const float kHighMeanEndHz = std::min(0.75f * kAliasingFreqHz, 2551897f77806baad82d321efb6c4c2f27f8c9458b3aluebs sample_rate_hz_ / 2.f); 2561897f77806baad82d321efb6c4c2f27f8c9458b3aluebs high_mean_start_bin_ = Round(kHighMeanStartHz * kFftSize / sample_rate_hz_); 2571897f77806baad82d321efb6c4c2f27f8c9458b3aluebs high_mean_end_bin_ = Round(kHighMeanEndHz * kFftSize / sample_rate_hz_); 258cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 2591897f77806baad82d321efb6c4c2f27f8c9458b3aluebs RTC_DCHECK_LT(low_mean_end_bin_, high_mean_end_bin_); 2601897f77806baad82d321efb6c4c2f27f8c9458b3aluebs RTC_DCHECK_LT(high_mean_start_bin_, high_mean_end_bin_); 2611897f77806baad82d321efb6c4c2f27f8c9458b3aluebs RTC_DCHECK_LT(high_mean_end_bin_, kNumFreqBins - 1); 2621897f77806baad82d321efb6c4c2f27f8c9458b3aluebs} 2631897f77806baad82d321efb6c4c2f27f8c9458b3aluebs 26445daf7b26f49793c30e395f7ba7be30aa51936bbaluebsvoid NonlinearBeamformer::InitInterfAngles() { 26545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs interf_angles_radians_.clear(); 266cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs const Point target_direction = AzimuthToPoint(target_angle_radians_); 267cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs const Point clockwise_interf_direction = 268cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs AzimuthToPoint(target_angle_radians_ - away_radians_); 269cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs if (!array_normal_ || 270cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs DotProduct(*array_normal_, target_direction) * 271cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs DotProduct(*array_normal_, clockwise_interf_direction) >= 272cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 0.f) { 273cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // The target and clockwise interferer are in the same half-plane defined 274cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // by the array. 275cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs interf_angles_radians_.push_back(target_angle_radians_ - away_radians_); 276cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } else { 277cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // Otherwise, the interferer will begin reflecting back at the target. 278cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // Instead rotate it away 180 degrees. 279cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs interf_angles_radians_.push_back(target_angle_radians_ - away_radians_ + 280cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs M_PI); 281cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } 282cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs const Point counterclock_interf_direction = 283cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs AzimuthToPoint(target_angle_radians_ + away_radians_); 284cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs if (!array_normal_ || 285cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs DotProduct(*array_normal_, target_direction) * 286cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs DotProduct(*array_normal_, counterclock_interf_direction) >= 287cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 0.f) { 288cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // The target and counter-clockwise interferer are in the same half-plane 289cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // defined by the array. 290cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs interf_angles_radians_.push_back(target_angle_radians_ + away_radians_); 291cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } else { 292cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // Otherwise, the interferer will begin reflecting back at the target. 293cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // Instead rotate it away 180 degrees. 294cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs interf_angles_radians_.push_back(target_angle_radians_ + away_radians_ - 295cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs M_PI); 296cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } 29745daf7b26f49793c30e395f7ba7be30aa51936bbaluebs} 29845daf7b26f49793c30e395f7ba7be30aa51936bbaluebs 2990f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::InitDelaySumMasks() { 300dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t f_ix = 0; f_ix < kNumFreqBins; ++f_ix) { 3010c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org delay_sum_masks_[f_ix].Resize(1, num_input_channels_); 302cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs CovarianceMatrixGenerator::PhaseAlignmentMasks( 303cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs f_ix, kFftSize, sample_rate_hz_, kSpeedOfSoundMeterSeconds, 304cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs array_geometry_, target_angle_radians_, &delay_sum_masks_[f_ix]); 3050c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 3060c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex_f norm_factor = sqrt( 3070c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org ConjugateDotProduct(delay_sum_masks_[f_ix], delay_sum_masks_[f_ix])); 3080c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org delay_sum_masks_[f_ix].Scale(1.f / norm_factor); 3092a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org normalized_delay_sum_masks_[f_ix].CopyFrom(delay_sum_masks_[f_ix]); 3102a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org normalized_delay_sum_masks_[f_ix].Scale(1.f / SumAbs( 3112a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org normalized_delay_sum_masks_[f_ix])); 3120c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 3130c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 3140c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 3150f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::InitTargetCovMats() { 316dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = 0; i < kNumFreqBins; ++i) { 3170c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org target_cov_mats_[i].Resize(num_input_channels_, num_input_channels_); 3181d88394bcb91ba74d7425678079731156e9ba281aluebs@webrtc.org TransposedConjugatedProduct(delay_sum_masks_[i], &target_cov_mats_[i]); 3190c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 3200c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 3210c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 322cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsvoid NonlinearBeamformer::InitDiffuseCovMats() { 323cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs for (size_t i = 0; i < kNumFreqBins; ++i) { 324cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs uniform_cov_mat_[i].Resize(num_input_channels_, num_input_channels_); 325cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs CovarianceMatrixGenerator::UniformCovarianceMatrix( 326cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs wave_numbers_[i], array_geometry_, &uniform_cov_mat_[i]); 327cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs complex_f normalization_factor = uniform_cov_mat_[i].elements()[0][0]; 328cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs uniform_cov_mat_[i].Scale(1.f / normalization_factor); 329cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs uniform_cov_mat_[i].Scale(1 - kBalance); 330cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } 331cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs} 332cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 3330f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::InitInterfCovMats() { 334dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = 0; i < kNumFreqBins; ++i) { 33545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs interf_cov_mats_[i].clear(); 33645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { 33745daf7b26f49793c30e395f7ba7be30aa51936bbaluebs interf_cov_mats_[i].push_back(new ComplexMatrixF(num_input_channels_, 33845daf7b26f49793c30e395f7ba7be30aa51936bbaluebs num_input_channels_)); 33945daf7b26f49793c30e395f7ba7be30aa51936bbaluebs ComplexMatrixF angled_cov_mat(num_input_channels_, num_input_channels_); 34045daf7b26f49793c30e395f7ba7be30aa51936bbaluebs CovarianceMatrixGenerator::AngledCovarianceMatrix( 34145daf7b26f49793c30e395f7ba7be30aa51936bbaluebs kSpeedOfSoundMeterSeconds, 34245daf7b26f49793c30e395f7ba7be30aa51936bbaluebs interf_angles_radians_[j], 34345daf7b26f49793c30e395f7ba7be30aa51936bbaluebs i, 34445daf7b26f49793c30e395f7ba7be30aa51936bbaluebs kFftSize, 34545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs kNumFreqBins, 34645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs sample_rate_hz_, 34745daf7b26f49793c30e395f7ba7be30aa51936bbaluebs array_geometry_, 34845daf7b26f49793c30e395f7ba7be30aa51936bbaluebs &angled_cov_mat); 34945daf7b26f49793c30e395f7ba7be30aa51936bbaluebs // Normalize matrices before averaging them. 350cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs complex_f normalization_factor = angled_cov_mat.elements()[0][0]; 35145daf7b26f49793c30e395f7ba7be30aa51936bbaluebs angled_cov_mat.Scale(1.f / normalization_factor); 35245daf7b26f49793c30e395f7ba7be30aa51936bbaluebs // Weighted average of matrices. 35345daf7b26f49793c30e395f7ba7be30aa51936bbaluebs angled_cov_mat.Scale(kBalance); 354cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs interf_cov_mats_[i][j]->Add(uniform_cov_mat_[i], angled_cov_mat); 355cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } 356cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs } 357cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs} 358cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 359cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsvoid NonlinearBeamformer::NormalizeCovMats() { 360cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs for (size_t i = 0; i < kNumFreqBins; ++i) { 361cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs rxiws_[i] = Norm(target_cov_mats_[i], delay_sum_masks_[i]); 362cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs rpsiws_[i].clear(); 363cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs for (size_t j = 0; j < interf_angles_radians_.size(); ++j) { 364cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs rpsiws_[i].push_back(Norm(*interf_cov_mats_[i][j], delay_sum_masks_[i])); 36545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs } 3660c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 3670c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 3680c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 369dfa36058c945cf2ef9932a566987f648c24fa632Michael Graczykvoid NonlinearBeamformer::ProcessChunk(const ChannelBuffer<float>& input, 370cb05b72eb2f7db4478b93b16faf31ec74237453eAndrew MacDonald ChannelBuffer<float>* output) { 37191d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK_EQ(input.num_channels(), num_input_channels_); 37291d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK_EQ(input.num_frames_per_band(), chunk_length_); 3730c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 3740c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float old_high_pass_mask = high_pass_postfilter_mask_; 375dfa36058c945cf2ef9932a566987f648c24fa632Michael Graczyk lapped_transform_->ProcessChunk(input.channels(0), output->channels(0)); 3763aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org // Ramp up/down for smoothing. 1 mask per 10ms results in audible 3773aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org // discontinuities. 3783aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org const float ramp_increment = 3793aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org (high_pass_postfilter_mask_ - old_high_pass_mask) / 380dfa36058c945cf2ef9932a566987f648c24fa632Michael Graczyk input.num_frames_per_band(); 381cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // Apply the smoothed high-pass mask to the first channel of each band. 38225702cb1628941427fa55e528f53483f239ae011pkasting // This can be done because the effect of the linear beamformer is negligible 383cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs // compared to the post-filter. 384dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = 1; i < input.num_bands(); ++i) { 3853aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org float smoothed_mask = old_high_pass_mask; 386dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t j = 0; j < input.num_frames_per_band(); ++j) { 3873aca0b0b312a77e9399502a64c27bc3fcb981927aluebs@webrtc.org smoothed_mask += ramp_increment; 388cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs output->channels(i)[0][j] = input.channels(i)[0][j] * smoothed_mask; 3890c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 3900c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 3910c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 3920c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 393cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebsvoid NonlinearBeamformer::AimAt(const SphericalPointf& target_direction) { 394cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs target_angle_radians_ = target_direction.azimuth(); 395cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitHighFrequencyCorrectionRanges(); 396cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitInterfAngles(); 397cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitDelaySumMasks(); 398cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitTargetCovMats(); 399cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs InitInterfCovMats(); 400cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs NormalizeCovMats(); 401cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs} 402cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs 4031adbacb19dbe6f4c537c9cc9d04242b61735a2b8blochbool NonlinearBeamformer::IsInBeam(const SphericalPointf& spherical_point) { 4041adbacb19dbe6f4c537c9cc9d04242b61735a2b8bloch // If more than half-beamwidth degrees away from the beam's center, 4051adbacb19dbe6f4c537c9cc9d04242b61735a2b8bloch // you are out of the beam. 406cb3f9bd9c024f11e1ee060de23bf65c7a1f9f594Alejandro Luebs return fabs(spherical_point.azimuth() - target_angle_radians_) < 4071adbacb19dbe6f4c537c9cc9d04242b61735a2b8bloch kHalfBeamWidthRadians; 4081adbacb19dbe6f4c537c9cc9d04242b61735a2b8bloch} 4091adbacb19dbe6f4c537c9cc9d04242b61735a2b8bloch 4100f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::ProcessAudioBlock(const complex_f* const* input, 4116955870806624479723addfae6dcf5d13968796cPeter Kasting size_t num_input_channels, 412dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting size_t num_freq_bins, 4136955870806624479723addfae6dcf5d13968796cPeter Kasting size_t num_output_channels, 414cb05b72eb2f7db4478b93b16faf31ec74237453eAndrew MacDonald complex_f* const* output) { 41525702cb1628941427fa55e528f53483f239ae011pkasting RTC_CHECK_EQ(kNumFreqBins, num_freq_bins); 41625702cb1628941427fa55e528f53483f239ae011pkasting RTC_CHECK_EQ(num_input_channels_, num_input_channels); 4176955870806624479723addfae6dcf5d13968796cPeter Kasting RTC_CHECK_EQ(1u, num_output_channels); 4180c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 419ec4521cdb488997ce19bc3ed8ddc4b327187e609aluebs@webrtc.org // Calculating the post-filter masks. Note that we need two for each 4200c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org // frequency bin to account for the positive and negative interferer 4210c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org // angle. 422dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { 4230c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org eig_m_.CopyFromColumn(input, i, num_input_channels_); 424661af50dd55a2beb488d4995ed21c5ffd53f8eddaluebs@webrtc.org float eig_m_norm_factor = std::sqrt(SumSquares(eig_m_)); 4250c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org if (eig_m_norm_factor != 0.f) { 4260c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org eig_m_.Scale(1.f / eig_m_norm_factor); 4270c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 4280c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 4290c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float rxim = Norm(target_cov_mats_[i], eig_m_); 4300c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float ratio_rxiw_rxim = 0.f; 43191ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org if (rxim > 0.f) { 4320c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org ratio_rxiw_rxim = rxiws_[i] / rxim; 4330c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 4340c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 4350c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex_f rmw = abs(ConjugateDotProduct(delay_sum_masks_[i], eig_m_)); 4360c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org rmw *= rmw; 4370c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float rmw_r = rmw.real(); 4380c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 43945daf7b26f49793c30e395f7ba7be30aa51936bbaluebs new_mask_[i] = CalculatePostfilterMask(*interf_cov_mats_[i][0], 44045daf7b26f49793c30e395f7ba7be30aa51936bbaluebs rpsiws_[i][0], 4410c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org ratio_rxiw_rxim, 44245daf7b26f49793c30e395f7ba7be30aa51936bbaluebs rmw_r); 44345daf7b26f49793c30e395f7ba7be30aa51936bbaluebs for (size_t j = 1; j < interf_angles_radians_.size(); ++j) { 44445daf7b26f49793c30e395f7ba7be30aa51936bbaluebs float tmp_mask = CalculatePostfilterMask(*interf_cov_mats_[i][j], 44545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs rpsiws_[i][j], 44645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs ratio_rxiw_rxim, 44745daf7b26f49793c30e395f7ba7be30aa51936bbaluebs rmw_r); 44845daf7b26f49793c30e395f7ba7be30aa51936bbaluebs if (tmp_mask < new_mask_[i]) { 44945daf7b26f49793c30e395f7ba7be30aa51936bbaluebs new_mask_[i] = tmp_mask; 45045daf7b26f49793c30e395f7ba7be30aa51936bbaluebs } 45145daf7b26f49793c30e395f7ba7be30aa51936bbaluebs } 4520c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 4530c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 454645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald ApplyMaskTimeSmoothing(); 455645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald EstimateTargetPresence(); 4560c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org ApplyLowFrequencyCorrection(); 457799e667e9fbf304cac628c12a955ff8c1aaf44ddaluebs@webrtc.org ApplyHighFrequencyCorrection(); 458645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald ApplyMaskFrequencySmoothing(); 4590c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org ApplyMasks(input, output); 4600c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 4610c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 4620f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgfloat NonlinearBeamformer::CalculatePostfilterMask( 4630f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.org const ComplexMatrixF& interf_cov_mat, 4640f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.org float rpsiw, 4650f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.org float ratio_rxiw_rxim, 46645daf7b26f49793c30e395f7ba7be30aa51936bbaluebs float rmw_r) { 4670c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org float rpsim = Norm(interf_cov_mat, eig_m_); 4680c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 46991ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org float ratio = 0.f; 47091ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org if (rpsim > 0.f) { 47191ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org ratio = rpsiw / rpsim; 47291ba79ae3f984c7e1b6f1a6fb61f9969ba236ec1aluebs@webrtc.org } 4730c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 47445daf7b26f49793c30e395f7ba7be30aa51936bbaluebs return (1.f - std::min(kCutOffConstant, ratio / rmw_r)) / 47545daf7b26f49793c30e395f7ba7be30aa51936bbaluebs (1.f - std::min(kCutOffConstant, ratio / ratio_rxiw_rxim)); 4760c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 4770c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 4780f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::ApplyMasks(const complex_f* const* input, 479cb05b72eb2f7db4478b93b16faf31ec74237453eAndrew MacDonald complex_f* const* output) { 4800c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org complex_f* output_channel = output[0]; 481dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t f_ix = 0; f_ix < kNumFreqBins; ++f_ix) { 4820c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org output_channel[f_ix] = complex_f(0.f, 0.f); 4830c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 4842a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org const complex_f* delay_sum_mask_els = 4852a44be93e86012bc9f91f849788a34d0734df249aluebs@webrtc.org normalized_delay_sum_masks_[f_ix].elements()[0]; 4866955870806624479723addfae6dcf5d13968796cPeter Kasting for (size_t c_ix = 0; c_ix < num_input_channels_; ++c_ix) { 4870c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org output_channel[f_ix] += input[c_ix][f_ix] * delay_sum_mask_els[c_ix]; 4880c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 4890c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 49045daf7b26f49793c30e395f7ba7be30aa51936bbaluebs output_channel[f_ix] *= kCompensationGain * final_mask_[f_ix]; 4910c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 4920c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 4930c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 494645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Smooth new_mask_ into time_smooth_mask_. 495645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldvoid NonlinearBeamformer::ApplyMaskTimeSmoothing() { 496dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = low_mean_start_bin_; i <= high_mean_end_bin_; ++i) { 497645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald time_smooth_mask_[i] = kMaskTimeSmoothAlpha * new_mask_[i] + 498645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald (1 - kMaskTimeSmoothAlpha) * time_smooth_mask_[i]; 4990c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 5000c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 5010c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 502645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Copy time_smooth_mask_ to final_mask_ and smooth over frequency. 503645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldvoid NonlinearBeamformer::ApplyMaskFrequencySmoothing() { 504645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // Smooth over frequency in both directions. The "frequency correction" 505645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // regions have constant value, but we enter them to smooth over the jump 506645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // that exists at the boundary. However, this does mean when smoothing "away" 507645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // from the region that we only need to use the last element. 508645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // 509645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // Upward smoothing: 510645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // low_mean_start_bin_ 511645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // v 512645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // |------|------------|------| 513645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // ^------------------>^ 514645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // 515645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // Downward smoothing: 516645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // high_mean_end_bin_ 517645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // v 518645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // |------|------------|------| 519645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald // ^<------------------^ 520645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald std::copy(time_smooth_mask_, time_smooth_mask_ + kNumFreqBins, final_mask_); 521dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = low_mean_start_bin_; i < kNumFreqBins; ++i) { 522645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald final_mask_[i] = kMaskFrequencySmoothAlpha * final_mask_[i] + 523645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald (1 - kMaskFrequencySmoothAlpha) * final_mask_[i - 1]; 5240c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 525dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting for (size_t i = high_mean_end_bin_ + 1; i > 0; --i) { 526b297c5a01f88219da26cffe433804963d1b70f0fpkasting final_mask_[i - 1] = kMaskFrequencySmoothAlpha * final_mask_[i - 1] + 527b297c5a01f88219da26cffe433804963d1b70f0fpkasting (1 - kMaskFrequencySmoothAlpha) * final_mask_[i]; 5280c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org } 5290c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 5300c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 531645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Apply low frequency correction to time_smooth_mask_. 532645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldvoid NonlinearBeamformer::ApplyLowFrequencyCorrection() { 533645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald const float low_frequency_mask = 534645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald MaskRangeMean(low_mean_start_bin_, low_mean_end_bin_ + 1); 535645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald std::fill(time_smooth_mask_, time_smooth_mask_ + low_mean_start_bin_, 536645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald low_frequency_mask); 537645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald} 5380c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 539645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Apply high frequency correction to time_smooth_mask_. Update 540645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// high_pass_postfilter_mask_ to use for the high frequency time-domain bands. 541645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonaldvoid NonlinearBeamformer::ApplyHighFrequencyCorrection() { 542645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald high_pass_postfilter_mask_ = 543645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald MaskRangeMean(high_mean_start_bin_, high_mean_end_bin_ + 1); 544645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald std::fill(time_smooth_mask_ + high_mean_end_bin_ + 1, 545645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald time_smooth_mask_ + kNumFreqBins, high_pass_postfilter_mask_); 546645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald} 547799e667e9fbf304cac628c12a955ff8c1aaf44ddaluebs@webrtc.org 548645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald// Compute mean over the given range of time_smooth_mask_, [first, last). 549dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kastingfloat NonlinearBeamformer::MaskRangeMean(size_t first, size_t last) { 55091d6edef35e7275879c30ce16ecb8b6dc73c6e4ahenrikg RTC_DCHECK_GT(last, first); 551645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald const float sum = std::accumulate(time_smooth_mask_ + first, 552645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald time_smooth_mask_ + last, 0.f); 553645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald return sum / (last - first); 5540c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} 5550c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org 5560f663de2ec25731d538a3a7d9b4d2c21ae61af8dmgraczyk@chromium.orgvoid NonlinearBeamformer::EstimateTargetPresence() { 557dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting const size_t quantile = static_cast<size_t>( 558645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald (high_mean_end_bin_ - low_mean_start_bin_) * kMaskQuantile + 559dce40cf804019a9898b6ab8d8262466b697c56e0Peter Kasting low_mean_start_bin_); 560645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald std::nth_element(new_mask_ + low_mean_start_bin_, new_mask_ + quantile, 561645299d4e092e6e3828621045afbc2cb311476c4Andrew MacDonald new_mask_ + high_mean_end_bin_ + 1); 56292a19bcbd7857a8fb368fa2309b148d304932a36aluebs@webrtc.org if (new_mask_[quantile] > kMaskTargetThreshold) { 563d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org is_target_present_ = true; 564d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org interference_blocks_count_ = 0; 565d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org } else { 566d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org is_target_present_ = interference_blocks_count_++ < hold_target_blocks_; 567d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org } 568d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org} 569d82f55d2a73da925d4160e3d1430f79f76170992aluebs@webrtc.org 5700c39e91cc827f222ab227bad09cf9199163d28f8aluebs@webrtc.org} // namespace webrtc 571