1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/aec3/render_delay_buffer.h"
12
13 #include <string.h>
14
15 #include <algorithm>
16 #include <cmath>
17 #include <memory>
18 #include <numeric>
19 #include <vector>
20
21 #include "absl/types/optional.h"
22 #include "api/array_view.h"
23 #include "api/audio/echo_canceller3_config.h"
24 #include "modules/audio_processing/aec3/aec3_common.h"
25 #include "modules/audio_processing/aec3/aec3_fft.h"
26 #include "modules/audio_processing/aec3/alignment_mixer.h"
27 #include "modules/audio_processing/aec3/block_buffer.h"
28 #include "modules/audio_processing/aec3/decimator.h"
29 #include "modules/audio_processing/aec3/downsampled_render_buffer.h"
30 #include "modules/audio_processing/aec3/fft_buffer.h"
31 #include "modules/audio_processing/aec3/fft_data.h"
32 #include "modules/audio_processing/aec3/render_buffer.h"
33 #include "modules/audio_processing/aec3/spectrum_buffer.h"
34 #include "modules/audio_processing/logging/apm_data_dumper.h"
35 #include "rtc_base/atomic_ops.h"
36 #include "rtc_base/checks.h"
37 #include "rtc_base/logging.h"
38
39 namespace webrtc {
40 namespace {
41
42 class RenderDelayBufferImpl final : public RenderDelayBuffer {
43 public:
44 RenderDelayBufferImpl(const EchoCanceller3Config& config,
45 int sample_rate_hz,
46 size_t num_render_channels);
47 RenderDelayBufferImpl() = delete;
48 ~RenderDelayBufferImpl() override;
49
50 void Reset() override;
51 BufferingEvent Insert(
52 const std::vector<std::vector<std::vector<float>>>& block) override;
53 BufferingEvent PrepareCaptureProcessing() override;
54 bool AlignFromDelay(size_t delay) override;
55 void AlignFromExternalDelay() override;
Delay() const56 size_t Delay() const override { return ComputeDelay(); }
MaxDelay() const57 size_t MaxDelay() const override {
58 return blocks_.buffer.size() - 1 - buffer_headroom_;
59 }
GetRenderBuffer()60 RenderBuffer* GetRenderBuffer() override { return &echo_remover_buffer_; }
61
GetDownsampledRenderBuffer() const62 const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override {
63 return low_rate_;
64 }
65
66 int BufferLatency() const;
67 void SetAudioBufferDelay(int delay_ms) override;
68 bool HasReceivedBufferDelay() override;
69
70 private:
71 static int instance_count_;
72 std::unique_ptr<ApmDataDumper> data_dumper_;
73 const Aec3Optimization optimization_;
74 const EchoCanceller3Config config_;
75 const float render_linear_amplitude_gain_;
76 const rtc::LoggingSeverity delay_log_level_;
77 size_t down_sampling_factor_;
78 const int sub_block_size_;
79 BlockBuffer blocks_;
80 SpectrumBuffer spectra_;
81 FftBuffer ffts_;
82 absl::optional<size_t> delay_;
83 RenderBuffer echo_remover_buffer_;
84 DownsampledRenderBuffer low_rate_;
85 AlignmentMixer render_mixer_;
86 Decimator render_decimator_;
87 const Aec3Fft fft_;
88 std::vector<float> render_ds_;
89 const int buffer_headroom_;
90 bool last_call_was_render_ = false;
91 int num_api_calls_in_a_row_ = 0;
92 int max_observed_jitter_ = 1;
93 int64_t capture_call_counter_ = 0;
94 int64_t render_call_counter_ = 0;
95 bool render_activity_ = false;
96 size_t render_activity_counter_ = 0;
97 absl::optional<int> external_audio_buffer_delay_;
98 bool external_audio_buffer_delay_verified_after_reset_ = false;
99 size_t min_latency_blocks_ = 0;
100 size_t excess_render_detection_counter_ = 0;
101
102 int MapDelayToTotalDelay(size_t delay) const;
103 int ComputeDelay() const;
104 void ApplyTotalDelay(int delay);
105 void InsertBlock(const std::vector<std::vector<std::vector<float>>>& block,
106 int previous_write);
107 bool DetectActiveRender(rtc::ArrayView<const float> x) const;
108 bool DetectExcessRenderBlocks();
109 void IncrementWriteIndices();
110 void IncrementLowRateReadIndices();
111 void IncrementReadIndices();
112 bool RenderOverrun();
113 bool RenderUnderrun();
114 };
115
116 int RenderDelayBufferImpl::instance_count_ = 0;
117
RenderDelayBufferImpl(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels)118 RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
119 int sample_rate_hz,
120 size_t num_render_channels)
121 : data_dumper_(
122 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
123 optimization_(DetectOptimization()),
124 config_(config),
125 render_linear_amplitude_gain_(
126 std::pow(10.0f, config_.render_levels.render_power_gain_db / 20.f)),
127 delay_log_level_(config_.delay.log_warning_on_delay_changes
128 ? rtc::LS_WARNING
129 : rtc::LS_VERBOSE),
130 down_sampling_factor_(config.delay.down_sampling_factor),
131 sub_block_size_(static_cast<int>(down_sampling_factor_ > 0
132 ? kBlockSize / down_sampling_factor_
133 : kBlockSize)),
134 blocks_(GetRenderDelayBufferSize(down_sampling_factor_,
135 config.delay.num_filters,
136 config.filter.refined.length_blocks),
137 NumBandsForRate(sample_rate_hz),
138 num_render_channels,
139 kBlockSize),
140 spectra_(blocks_.buffer.size(), num_render_channels),
141 ffts_(blocks_.buffer.size(), num_render_channels),
142 delay_(config_.delay.default_delay),
143 echo_remover_buffer_(&blocks_, &spectra_, &ffts_),
144 low_rate_(GetDownSampledBufferSize(down_sampling_factor_,
145 config.delay.num_filters)),
146 render_mixer_(num_render_channels, config.delay.render_alignment_mixing),
147 render_decimator_(down_sampling_factor_),
148 fft_(),
149 render_ds_(sub_block_size_, 0.f),
150 buffer_headroom_(config.filter.refined.length_blocks) {
151 RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size());
152 RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size());
153 for (size_t i = 0; i < blocks_.buffer.size(); ++i) {
154 RTC_DCHECK_EQ(blocks_.buffer[i][0].size(), ffts_.buffer[i].size());
155 RTC_DCHECK_EQ(spectra_.buffer[i].size(), ffts_.buffer[i].size());
156 }
157
158 Reset();
159 }
160
161 RenderDelayBufferImpl::~RenderDelayBufferImpl() = default;
162
163 // Resets the buffer delays and clears the reported delays.
Reset()164 void RenderDelayBufferImpl::Reset() {
165 last_call_was_render_ = false;
166 num_api_calls_in_a_row_ = 1;
167 min_latency_blocks_ = 0;
168 excess_render_detection_counter_ = 0;
169
170 // Initialize the read index to one sub-block before the write index.
171 low_rate_.read = low_rate_.OffsetIndex(low_rate_.write, sub_block_size_);
172
173 // Check for any external audio buffer delay and whether it is feasible.
174 if (external_audio_buffer_delay_) {
175 const int headroom = 2;
176 size_t audio_buffer_delay_to_set;
177 // Minimum delay is 1 (like the low-rate render buffer).
178 if (*external_audio_buffer_delay_ <= headroom) {
179 audio_buffer_delay_to_set = 1;
180 } else {
181 audio_buffer_delay_to_set = *external_audio_buffer_delay_ - headroom;
182 }
183
184 audio_buffer_delay_to_set = std::min(audio_buffer_delay_to_set, MaxDelay());
185
186 // When an external delay estimate is available, use that delay as the
187 // initial render buffer delay.
188 ApplyTotalDelay(audio_buffer_delay_to_set);
189 delay_ = ComputeDelay();
190
191 external_audio_buffer_delay_verified_after_reset_ = false;
192 } else {
193 // If an external delay estimate is not available, use that delay as the
194 // initial delay. Set the render buffer delays to the default delay.
195 ApplyTotalDelay(config_.delay.default_delay);
196
197 // Unset the delays which are set by AlignFromDelay.
198 delay_ = absl::nullopt;
199 }
200 }
201
202 // Inserts a new block into the render buffers.
Insert(const std::vector<std::vector<std::vector<float>>> & block)203 RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert(
204 const std::vector<std::vector<std::vector<float>>>& block) {
205 ++render_call_counter_;
206 if (delay_) {
207 if (!last_call_was_render_) {
208 last_call_was_render_ = true;
209 num_api_calls_in_a_row_ = 1;
210 } else {
211 if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
212 max_observed_jitter_ = num_api_calls_in_a_row_;
213 RTC_LOG_V(delay_log_level_)
214 << "New max number api jitter observed at render block "
215 << render_call_counter_ << ": " << num_api_calls_in_a_row_
216 << " blocks";
217 }
218 }
219 }
220
221 // Increase the write indices to where the new blocks should be written.
222 const int previous_write = blocks_.write;
223 IncrementWriteIndices();
224
225 // Allow overrun and do a reset when render overrun occurrs due to more render
226 // data being inserted than capture data is received.
227 BufferingEvent event =
228 RenderOverrun() ? BufferingEvent::kRenderOverrun : BufferingEvent::kNone;
229
230 // Detect and update render activity.
231 if (!render_activity_) {
232 render_activity_counter_ += DetectActiveRender(block[0][0]) ? 1 : 0;
233 render_activity_ = render_activity_counter_ >= 20;
234 }
235
236 // Insert the new render block into the specified position.
237 InsertBlock(block, previous_write);
238
239 if (event != BufferingEvent::kNone) {
240 Reset();
241 }
242
243 return event;
244 }
245
246 // Prepares the render buffers for processing another capture block.
247 RenderDelayBuffer::BufferingEvent
PrepareCaptureProcessing()248 RenderDelayBufferImpl::PrepareCaptureProcessing() {
249 RenderDelayBuffer::BufferingEvent event = BufferingEvent::kNone;
250 ++capture_call_counter_;
251
252 if (delay_) {
253 if (last_call_was_render_) {
254 last_call_was_render_ = false;
255 num_api_calls_in_a_row_ = 1;
256 } else {
257 if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
258 max_observed_jitter_ = num_api_calls_in_a_row_;
259 RTC_LOG_V(delay_log_level_)
260 << "New max number api jitter observed at capture block "
261 << capture_call_counter_ << ": " << num_api_calls_in_a_row_
262 << " blocks";
263 }
264 }
265 }
266
267 if (DetectExcessRenderBlocks()) {
268 // Too many render blocks compared to capture blocks. Risk of delay ending
269 // up before the filter used by the delay estimator.
270 RTC_LOG_V(delay_log_level_)
271 << "Excess render blocks detected at block " << capture_call_counter_;
272 Reset();
273 event = BufferingEvent::kRenderOverrun;
274 } else if (RenderUnderrun()) {
275 // Don't increment the read indices of the low rate buffer if there is a
276 // render underrun.
277 RTC_LOG_V(delay_log_level_)
278 << "Render buffer underrun detected at block " << capture_call_counter_;
279 IncrementReadIndices();
280 // Incrementing the buffer index without increasing the low rate buffer
281 // index means that the delay is reduced by one.
282 if (delay_ && *delay_ > 0)
283 delay_ = *delay_ - 1;
284 event = BufferingEvent::kRenderUnderrun;
285 } else {
286 // Increment the read indices in the render buffers to point to the most
287 // recent block to use in the capture processing.
288 IncrementLowRateReadIndices();
289 IncrementReadIndices();
290 }
291
292 echo_remover_buffer_.SetRenderActivity(render_activity_);
293 if (render_activity_) {
294 render_activity_counter_ = 0;
295 render_activity_ = false;
296 }
297
298 return event;
299 }
300
301 // Sets the delay and returns a bool indicating whether the delay was changed.
AlignFromDelay(size_t delay)302 bool RenderDelayBufferImpl::AlignFromDelay(size_t delay) {
303 RTC_DCHECK(!config_.delay.use_external_delay_estimator);
304 if (!external_audio_buffer_delay_verified_after_reset_ &&
305 external_audio_buffer_delay_ && delay_) {
306 int difference = static_cast<int>(delay) - static_cast<int>(*delay_);
307 RTC_LOG_V(delay_log_level_)
308 << "Mismatch between first estimated delay after reset "
309 "and externally reported audio buffer delay: "
310 << difference << " blocks";
311 external_audio_buffer_delay_verified_after_reset_ = true;
312 }
313 if (delay_ && *delay_ == delay) {
314 return false;
315 }
316 delay_ = delay;
317
318 // Compute the total delay and limit the delay to the allowed range.
319 int total_delay = MapDelayToTotalDelay(*delay_);
320 total_delay =
321 std::min(MaxDelay(), static_cast<size_t>(std::max(total_delay, 0)));
322
323 // Apply the delay to the buffers.
324 ApplyTotalDelay(total_delay);
325 return true;
326 }
327
SetAudioBufferDelay(int delay_ms)328 void RenderDelayBufferImpl::SetAudioBufferDelay(int delay_ms) {
329 if (!external_audio_buffer_delay_) {
330 RTC_LOG_V(delay_log_level_)
331 << "Receiving a first externally reported audio buffer delay of "
332 << delay_ms << " ms.";
333 }
334
335 // Convert delay from milliseconds to blocks (rounded down).
336 external_audio_buffer_delay_ = delay_ms / 4;
337 }
338
HasReceivedBufferDelay()339 bool RenderDelayBufferImpl::HasReceivedBufferDelay() {
340 return external_audio_buffer_delay_.has_value();
341 }
342
343 // Maps the externally computed delay to the delay used internally.
MapDelayToTotalDelay(size_t external_delay_blocks) const344 int RenderDelayBufferImpl::MapDelayToTotalDelay(
345 size_t external_delay_blocks) const {
346 const int latency_blocks = BufferLatency();
347 return latency_blocks + static_cast<int>(external_delay_blocks);
348 }
349
350 // Returns the delay (not including call jitter).
ComputeDelay() const351 int RenderDelayBufferImpl::ComputeDelay() const {
352 const int latency_blocks = BufferLatency();
353 int internal_delay = spectra_.read >= spectra_.write
354 ? spectra_.read - spectra_.write
355 : spectra_.size + spectra_.read - spectra_.write;
356
357 return internal_delay - latency_blocks;
358 }
359
360 // Set the read indices according to the delay.
ApplyTotalDelay(int delay)361 void RenderDelayBufferImpl::ApplyTotalDelay(int delay) {
362 RTC_LOG_V(delay_log_level_)
363 << "Applying total delay of " << delay << " blocks.";
364 blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay);
365 spectra_.read = spectra_.OffsetIndex(spectra_.write, delay);
366 ffts_.read = ffts_.OffsetIndex(ffts_.write, delay);
367 }
368
AlignFromExternalDelay()369 void RenderDelayBufferImpl::AlignFromExternalDelay() {
370 RTC_DCHECK(config_.delay.use_external_delay_estimator);
371 if (external_audio_buffer_delay_) {
372 int64_t delay = render_call_counter_ - capture_call_counter_ +
373 *external_audio_buffer_delay_;
374 ApplyTotalDelay(delay);
375 }
376 }
377
378 // Inserts a block into the render buffers.
InsertBlock(const std::vector<std::vector<std::vector<float>>> & block,int previous_write)379 void RenderDelayBufferImpl::InsertBlock(
380 const std::vector<std::vector<std::vector<float>>>& block,
381 int previous_write) {
382 auto& b = blocks_;
383 auto& lr = low_rate_;
384 auto& ds = render_ds_;
385 auto& f = ffts_;
386 auto& s = spectra_;
387 const size_t num_bands = b.buffer[b.write].size();
388 const size_t num_render_channels = b.buffer[b.write][0].size();
389 RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size());
390 for (size_t band = 0; band < num_bands; ++band) {
391 RTC_DCHECK_EQ(block[band].size(), num_render_channels);
392 RTC_DCHECK_EQ(b.buffer[b.write][band].size(), num_render_channels);
393 for (size_t ch = 0; ch < num_render_channels; ++ch) {
394 RTC_DCHECK_EQ(block[band][ch].size(), b.buffer[b.write][band][ch].size());
395 std::copy(block[band][ch].begin(), block[band][ch].end(),
396 b.buffer[b.write][band][ch].begin());
397 }
398 }
399
400 if (render_linear_amplitude_gain_ != 1.f) {
401 for (size_t band = 0; band < num_bands; ++band) {
402 for (size_t ch = 0; ch < num_render_channels; ++ch) {
403 for (size_t k = 0; k < 64; ++k) {
404 b.buffer[b.write][band][ch][k] *= render_linear_amplitude_gain_;
405 }
406 }
407 }
408 }
409
410 std::array<float, kBlockSize> downmixed_render;
411 render_mixer_.ProduceOutput(b.buffer[b.write][0], downmixed_render);
412 render_decimator_.Decimate(downmixed_render, ds);
413 data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(),
414 16000 / down_sampling_factor_, 1);
415 std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write);
416 for (size_t channel = 0; channel < b.buffer[b.write][0].size(); ++channel) {
417 fft_.PaddedFft(b.buffer[b.write][0][channel],
418 b.buffer[previous_write][0][channel],
419 &f.buffer[f.write][channel]);
420 f.buffer[f.write][channel].Spectrum(optimization_,
421 s.buffer[s.write][channel]);
422 }
423 }
424
DetectActiveRender(rtc::ArrayView<const float> x) const425 bool RenderDelayBufferImpl::DetectActiveRender(
426 rtc::ArrayView<const float> x) const {
427 const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f);
428 return x_energy > (config_.render_levels.active_render_limit *
429 config_.render_levels.active_render_limit) *
430 kFftLengthBy2;
431 }
432
DetectExcessRenderBlocks()433 bool RenderDelayBufferImpl::DetectExcessRenderBlocks() {
434 bool excess_render_detected = false;
435 const size_t latency_blocks = static_cast<size_t>(BufferLatency());
436 // The recently seen minimum latency in blocks. Should be close to 0.
437 min_latency_blocks_ = std::min(min_latency_blocks_, latency_blocks);
438 // After processing a configurable number of blocks the minimum latency is
439 // checked.
440 if (++excess_render_detection_counter_ >=
441 config_.buffering.excess_render_detection_interval_blocks) {
442 // If the minimum latency is not lower than the threshold there have been
443 // more render than capture frames.
444 excess_render_detected = min_latency_blocks_ >
445 config_.buffering.max_allowed_excess_render_blocks;
446 // Reset the counter and let the minimum latency be the current latency.
447 min_latency_blocks_ = latency_blocks;
448 excess_render_detection_counter_ = 0;
449 }
450
451 data_dumper_->DumpRaw("aec3_latency_blocks", latency_blocks);
452 data_dumper_->DumpRaw("aec3_min_latency_blocks", min_latency_blocks_);
453 data_dumper_->DumpRaw("aec3_excess_render_detected", excess_render_detected);
454 return excess_render_detected;
455 }
456
457 // Computes the latency in the buffer (the number of unread sub-blocks).
BufferLatency() const458 int RenderDelayBufferImpl::BufferLatency() const {
459 const DownsampledRenderBuffer& l = low_rate_;
460 int latency_samples = (l.buffer.size() + l.read - l.write) % l.buffer.size();
461 int latency_blocks = latency_samples / sub_block_size_;
462 return latency_blocks;
463 }
464
465 // Increments the write indices for the render buffers.
IncrementWriteIndices()466 void RenderDelayBufferImpl::IncrementWriteIndices() {
467 low_rate_.UpdateWriteIndex(-sub_block_size_);
468 blocks_.IncWriteIndex();
469 spectra_.DecWriteIndex();
470 ffts_.DecWriteIndex();
471 }
472
473 // Increments the read indices of the low rate render buffers.
IncrementLowRateReadIndices()474 void RenderDelayBufferImpl::IncrementLowRateReadIndices() {
475 low_rate_.UpdateReadIndex(-sub_block_size_);
476 }
477
478 // Increments the read indices for the render buffers.
IncrementReadIndices()479 void RenderDelayBufferImpl::IncrementReadIndices() {
480 if (blocks_.read != blocks_.write) {
481 blocks_.IncReadIndex();
482 spectra_.DecReadIndex();
483 ffts_.DecReadIndex();
484 }
485 }
486
487 // Checks for a render buffer overrun.
RenderOverrun()488 bool RenderDelayBufferImpl::RenderOverrun() {
489 return low_rate_.read == low_rate_.write || blocks_.read == blocks_.write;
490 }
491
492 // Checks for a render buffer underrun.
RenderUnderrun()493 bool RenderDelayBufferImpl::RenderUnderrun() {
494 return low_rate_.read == low_rate_.write;
495 }
496
497 } // namespace
498
Create(const EchoCanceller3Config & config,int sample_rate_hz,size_t num_render_channels)499 RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config,
500 int sample_rate_hz,
501 size_t num_render_channels) {
502 return new RenderDelayBufferImpl(config, sample_rate_hz, num_render_channels);
503 }
504
505 } // namespace webrtc
506