1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/aec3/stationarity_estimator.h"
12
13 #include <algorithm>
14 #include <array>
15
16 #include "api/array_view.h"
17 #include "modules/audio_processing/aec3/aec3_common.h"
18 #include "modules/audio_processing/aec3/spectrum_buffer.h"
19 #include "modules/audio_processing/logging/apm_data_dumper.h"
20 #include "rtc_base/atomic_ops.h"
21
22 namespace webrtc {
23
24 namespace {
25 constexpr float kMinNoisePower = 10.f;
26 constexpr int kHangoverBlocks = kNumBlocksPerSecond / 20;
27 constexpr int kNBlocksAverageInitPhase = 20;
28 constexpr int kNBlocksInitialPhase = kNumBlocksPerSecond * 2.;
29 } // namespace
30
StationarityEstimator()31 StationarityEstimator::StationarityEstimator()
32 : data_dumper_(
33 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) {
34 Reset();
35 }
36
37 StationarityEstimator::~StationarityEstimator() = default;
38
Reset()39 void StationarityEstimator::Reset() {
40 noise_.Reset();
41 hangovers_.fill(0);
42 stationarity_flags_.fill(false);
43 }
44
45 // Update just the noise estimator. Usefull until the delay is known
UpdateNoiseEstimator(rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> spectrum)46 void StationarityEstimator::UpdateNoiseEstimator(
47 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> spectrum) {
48 noise_.Update(spectrum);
49 data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum());
50 data_dumper_->DumpRaw("aec3_stationarity_is_block_stationary",
51 IsBlockStationary());
52 }
53
UpdateStationarityFlags(const SpectrumBuffer & spectrum_buffer,rtc::ArrayView<const float> render_reverb_contribution_spectrum,int idx_current,int num_lookahead)54 void StationarityEstimator::UpdateStationarityFlags(
55 const SpectrumBuffer& spectrum_buffer,
56 rtc::ArrayView<const float> render_reverb_contribution_spectrum,
57 int idx_current,
58 int num_lookahead) {
59 std::array<int, kWindowLength> indexes;
60 int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1);
61 int idx = idx_current;
62
63 if (num_lookahead_bounded < kWindowLength - 1) {
64 int num_lookback = (kWindowLength - 1) - num_lookahead_bounded;
65 idx = spectrum_buffer.OffsetIndex(idx_current, num_lookback);
66 }
67 // For estimating the stationarity properties of the current frame, the
68 // power for each band is accumulated for several consecutive spectra in the
69 // method EstimateBandStationarity.
70 // In order to avoid getting the indexes of the spectra for every band with
71 // its associated overhead, those indexes are stored in an array and then use
72 // when the estimation is done.
73 indexes[0] = idx;
74 for (size_t k = 1; k < indexes.size(); ++k) {
75 indexes[k] = spectrum_buffer.DecIndex(indexes[k - 1]);
76 }
77 RTC_DCHECK_EQ(
78 spectrum_buffer.DecIndex(indexes[kWindowLength - 1]),
79 spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1)));
80
81 for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
82 stationarity_flags_[k] = EstimateBandStationarity(
83 spectrum_buffer, render_reverb_contribution_spectrum, indexes, k);
84 }
85 UpdateHangover();
86 SmoothStationaryPerFreq();
87 }
88
IsBlockStationary() const89 bool StationarityEstimator::IsBlockStationary() const {
90 float acum_stationarity = 0.f;
91 RTC_DCHECK_EQ(stationarity_flags_.size(), kFftLengthBy2Plus1);
92 for (size_t band = 0; band < stationarity_flags_.size(); ++band) {
93 bool st = IsBandStationary(band);
94 acum_stationarity += static_cast<float>(st);
95 }
96 return ((acum_stationarity * (1.f / kFftLengthBy2Plus1)) > 0.75f);
97 }
98
EstimateBandStationarity(const SpectrumBuffer & spectrum_buffer,rtc::ArrayView<const float> average_reverb,const std::array<int,kWindowLength> & indexes,size_t band) const99 bool StationarityEstimator::EstimateBandStationarity(
100 const SpectrumBuffer& spectrum_buffer,
101 rtc::ArrayView<const float> average_reverb,
102 const std::array<int, kWindowLength>& indexes,
103 size_t band) const {
104 constexpr float kThrStationarity = 10.f;
105 float acum_power = 0.f;
106 const int num_render_channels =
107 static_cast<int>(spectrum_buffer.buffer[0].size());
108 const float one_by_num_channels = 1.f / num_render_channels;
109 for (auto idx : indexes) {
110 for (int ch = 0; ch < num_render_channels; ++ch) {
111 acum_power += spectrum_buffer.buffer[idx][ch][band] * one_by_num_channels;
112 }
113 }
114 acum_power += average_reverb[band];
115 float noise = kWindowLength * GetStationarityPowerBand(band);
116 RTC_CHECK_LT(0.f, noise);
117 bool stationary = acum_power < kThrStationarity * noise;
118 data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acum_power / noise);
119 return stationary;
120 }
121
AreAllBandsStationary()122 bool StationarityEstimator::AreAllBandsStationary() {
123 for (auto b : stationarity_flags_) {
124 if (!b)
125 return false;
126 }
127 return true;
128 }
129
UpdateHangover()130 void StationarityEstimator::UpdateHangover() {
131 bool reduce_hangover = AreAllBandsStationary();
132 for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
133 if (!stationarity_flags_[k]) {
134 hangovers_[k] = kHangoverBlocks;
135 } else if (reduce_hangover) {
136 hangovers_[k] = std::max(hangovers_[k] - 1, 0);
137 }
138 }
139 }
140
SmoothStationaryPerFreq()141 void StationarityEstimator::SmoothStationaryPerFreq() {
142 std::array<bool, kFftLengthBy2Plus1> all_ahead_stationary_smooth;
143 for (size_t k = 1; k < kFftLengthBy2Plus1 - 1; ++k) {
144 all_ahead_stationary_smooth[k] = stationarity_flags_[k - 1] &&
145 stationarity_flags_[k] &&
146 stationarity_flags_[k + 1];
147 }
148
149 all_ahead_stationary_smooth[0] = all_ahead_stationary_smooth[1];
150 all_ahead_stationary_smooth[kFftLengthBy2Plus1 - 1] =
151 all_ahead_stationary_smooth[kFftLengthBy2Plus1 - 2];
152
153 stationarity_flags_ = all_ahead_stationary_smooth;
154 }
155
156 int StationarityEstimator::instance_count_ = 0;
157
NoiseSpectrum()158 StationarityEstimator::NoiseSpectrum::NoiseSpectrum() {
159 Reset();
160 }
161
162 StationarityEstimator::NoiseSpectrum::~NoiseSpectrum() = default;
163
Reset()164 void StationarityEstimator::NoiseSpectrum::Reset() {
165 block_counter_ = 0;
166 noise_spectrum_.fill(kMinNoisePower);
167 }
168
Update(rtc::ArrayView<const std::array<float,kFftLengthBy2Plus1>> spectrum)169 void StationarityEstimator::NoiseSpectrum::Update(
170 rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> spectrum) {
171 RTC_DCHECK_LE(1, spectrum[0].size());
172 const int num_render_channels = static_cast<int>(spectrum.size());
173
174 std::array<float, kFftLengthBy2Plus1> avg_spectrum_data;
175 rtc::ArrayView<const float> avg_spectrum;
176 if (num_render_channels == 1) {
177 avg_spectrum = spectrum[0];
178 } else {
179 // For multiple channels, average the channel spectra before passing to the
180 // noise spectrum estimator.
181 avg_spectrum = avg_spectrum_data;
182 std::copy(spectrum[0].begin(), spectrum[0].end(),
183 avg_spectrum_data.begin());
184 for (int ch = 1; ch < num_render_channels; ++ch) {
185 for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) {
186 avg_spectrum_data[k] += spectrum[ch][k];
187 }
188 }
189
190 const float one_by_num_channels = 1.f / num_render_channels;
191 for (size_t k = 1; k < kFftLengthBy2Plus1; ++k) {
192 avg_spectrum_data[k] *= one_by_num_channels;
193 }
194 }
195
196 ++block_counter_;
197 float alpha = GetAlpha();
198 for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) {
199 if (block_counter_ <= kNBlocksAverageInitPhase) {
200 noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * avg_spectrum[k];
201 } else {
202 noise_spectrum_[k] =
203 UpdateBandBySmoothing(avg_spectrum[k], noise_spectrum_[k], alpha);
204 }
205 }
206 }
207
GetAlpha() const208 float StationarityEstimator::NoiseSpectrum::GetAlpha() const {
209 constexpr float kAlpha = 0.004f;
210 constexpr float kAlphaInit = 0.04f;
211 constexpr float kTiltAlpha = (kAlphaInit - kAlpha) / kNBlocksInitialPhase;
212
213 if (block_counter_ > (kNBlocksInitialPhase + kNBlocksAverageInitPhase)) {
214 return kAlpha;
215 } else {
216 return kAlphaInit -
217 kTiltAlpha * (block_counter_ - kNBlocksAverageInitPhase);
218 }
219 }
220
UpdateBandBySmoothing(float power_band,float power_band_noise,float alpha) const221 float StationarityEstimator::NoiseSpectrum::UpdateBandBySmoothing(
222 float power_band,
223 float power_band_noise,
224 float alpha) const {
225 float power_band_noise_updated = power_band_noise;
226 if (power_band_noise < power_band) {
227 RTC_DCHECK_GT(power_band, 0.f);
228 float alpha_inc = alpha * (power_band_noise / power_band);
229 if (block_counter_ > kNBlocksInitialPhase) {
230 if (10.f * power_band_noise < power_band) {
231 alpha_inc *= 0.1f;
232 }
233 }
234 power_band_noise_updated += alpha_inc * (power_band - power_band_noise);
235 } else {
236 power_band_noise_updated += alpha * (power_band - power_band_noise);
237 power_band_noise_updated =
238 std::max(power_band_noise_updated, kMinNoisePower);
239 }
240 return power_band_noise_updated;
241 }
242
243 } // namespace webrtc
244