1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/test/audio_processing_simulator.h"
12
13 #include <algorithm>
14 #include <fstream>
15 #include <iostream>
16 #include <memory>
17 #include <string>
18 #include <utility>
19 #include <vector>
20
21 #include "api/audio/echo_canceller3_config_json.h"
22 #include "api/audio/echo_canceller3_factory.h"
23 #include "modules/audio_processing/aec_dump/aec_dump_factory.h"
24 #include "modules/audio_processing/echo_control_mobile_impl.h"
25 #include "modules/audio_processing/include/audio_processing.h"
26 #include "modules/audio_processing/logging/apm_data_dumper.h"
27 #include "modules/audio_processing/test/fake_recording_device.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/logging.h"
30 #include "rtc_base/strings/json.h"
31 #include "rtc_base/strings/string_builder.h"
32
33 namespace webrtc {
34 namespace test {
35 namespace {
36 // Helper for reading JSON from a file and parsing it to an AEC3 configuration.
ReadAec3ConfigFromJsonFile(const std::string & filename)37 EchoCanceller3Config ReadAec3ConfigFromJsonFile(const std::string& filename) {
38 std::string json_string;
39 std::string s;
40 std::ifstream f(filename.c_str());
41 if (f.fail()) {
42 std::cout << "Failed to open the file " << filename << std::endl;
43 RTC_CHECK(false);
44 }
45 while (std::getline(f, s)) {
46 json_string += s;
47 }
48
49 bool parsing_successful;
50 EchoCanceller3Config cfg;
51 Aec3ConfigFromJsonString(json_string, &cfg, &parsing_successful);
52 if (!parsing_successful) {
53 std::cout << "Parsing of json string failed: " << std::endl
54 << json_string << std::endl;
55 RTC_CHECK(false);
56 }
57 RTC_CHECK(EchoCanceller3Config::Validate(&cfg));
58
59 return cfg;
60 }
61
62
GetIndexedOutputWavFilename(const std::string & wav_name,int counter)63 std::string GetIndexedOutputWavFilename(const std::string& wav_name,
64 int counter) {
65 rtc::StringBuilder ss;
66 ss << wav_name.substr(0, wav_name.size() - 4) << "_" << counter
67 << wav_name.substr(wav_name.size() - 4);
68 return ss.Release();
69 }
70
WriteEchoLikelihoodGraphFileHeader(std::ofstream * output_file)71 void WriteEchoLikelihoodGraphFileHeader(std::ofstream* output_file) {
72 (*output_file) << "import numpy as np" << std::endl
73 << "import matplotlib.pyplot as plt" << std::endl
74 << "y = np.array([";
75 }
76
WriteEchoLikelihoodGraphFileFooter(std::ofstream * output_file)77 void WriteEchoLikelihoodGraphFileFooter(std::ofstream* output_file) {
78 (*output_file) << "])" << std::endl
79 << "if __name__ == '__main__':" << std::endl
80 << " x = np.arange(len(y))*.01" << std::endl
81 << " plt.plot(x, y)" << std::endl
82 << " plt.ylabel('Echo likelihood')" << std::endl
83 << " plt.xlabel('Time (s)')" << std::endl
84 << " plt.show()" << std::endl;
85 }
86
87 // RAII class for execution time measurement. Updates the provided
88 // ApiCallStatistics based on the time between ScopedTimer creation and
89 // leaving the enclosing scope.
90 class ScopedTimer {
91 public:
ScopedTimer(ApiCallStatistics * api_call_statistics_,ApiCallStatistics::CallType call_type)92 ScopedTimer(ApiCallStatistics* api_call_statistics_,
93 ApiCallStatistics::CallType call_type)
94 : start_time_(rtc::TimeNanos()),
95 call_type_(call_type),
96 api_call_statistics_(api_call_statistics_) {}
97
~ScopedTimer()98 ~ScopedTimer() {
99 api_call_statistics_->Add(rtc::TimeNanos() - start_time_, call_type_);
100 }
101
102 private:
103 const int64_t start_time_;
104 const ApiCallStatistics::CallType call_type_;
105 ApiCallStatistics* const api_call_statistics_;
106 };
107
108 } // namespace
109
110 SimulationSettings::SimulationSettings() = default;
111 SimulationSettings::SimulationSettings(const SimulationSettings&) = default;
112 SimulationSettings::~SimulationSettings() = default;
113
AudioProcessingSimulator(const SimulationSettings & settings,rtc::scoped_refptr<AudioProcessing> audio_processing,std::unique_ptr<AudioProcessingBuilder> ap_builder)114 AudioProcessingSimulator::AudioProcessingSimulator(
115 const SimulationSettings& settings,
116 rtc::scoped_refptr<AudioProcessing> audio_processing,
117 std::unique_ptr<AudioProcessingBuilder> ap_builder)
118 : settings_(settings),
119 ap_(std::move(audio_processing)),
120 analog_mic_level_(settings.initial_mic_level),
121 fake_recording_device_(
122 settings.initial_mic_level,
123 settings_.simulate_mic_gain ? *settings.simulated_mic_kind : 0),
124 worker_queue_("file_writer_task_queue") {
125 RTC_CHECK(!settings_.dump_internal_data || WEBRTC_APM_DEBUG_DUMP == 1);
126 ApmDataDumper::SetActivated(settings_.dump_internal_data);
127 if (settings_.dump_internal_data_output_dir.has_value()) {
128 ApmDataDumper::SetOutputDirectory(
129 settings_.dump_internal_data_output_dir.value());
130 }
131
132 if (settings_.ed_graph_output_filename &&
133 !settings_.ed_graph_output_filename->empty()) {
134 residual_echo_likelihood_graph_writer_.open(
135 *settings_.ed_graph_output_filename);
136 RTC_CHECK(residual_echo_likelihood_graph_writer_.is_open());
137 WriteEchoLikelihoodGraphFileHeader(&residual_echo_likelihood_graph_writer_);
138 }
139
140 if (settings_.simulate_mic_gain)
141 RTC_LOG(LS_VERBOSE) << "Simulating analog mic gain";
142
143 // Create the audio processing object.
144 RTC_CHECK(!(ap_ && ap_builder))
145 << "The AudioProcessing and the AudioProcessingBuilder cannot both be "
146 "specified at the same time.";
147
148 if (ap_) {
149 RTC_CHECK(!settings_.aec_settings_filename);
150 RTC_CHECK(!settings_.print_aec_parameter_values);
151 } else {
152 // Use specied builder if such is provided, otherwise create a new builder.
153 std::unique_ptr<AudioProcessingBuilder> builder =
154 !!ap_builder ? std::move(ap_builder)
155 : std::make_unique<AudioProcessingBuilder>();
156
157 // Create and set an EchoCanceller3Factory if needed.
158 const bool use_aec = settings_.use_aec && *settings_.use_aec;
159 if (use_aec) {
160 EchoCanceller3Config cfg;
161 if (settings_.aec_settings_filename) {
162 if (settings_.use_verbose_logging) {
163 std::cout << "Reading AEC Parameters from JSON input." << std::endl;
164 }
165 cfg = ReadAec3ConfigFromJsonFile(*settings_.aec_settings_filename);
166 }
167
168 if (settings_.linear_aec_output_filename) {
169 cfg.filter.export_linear_aec_output = true;
170 }
171
172 if (settings_.print_aec_parameter_values) {
173 if (!settings_.use_quiet_output) {
174 std::cout << "AEC settings:" << std::endl;
175 }
176 std::cout << Aec3ConfigToJsonString(cfg) << std::endl;
177 }
178
179 auto echo_control_factory = std::make_unique<EchoCanceller3Factory>(cfg);
180 builder->SetEchoControlFactory(std::move(echo_control_factory));
181 }
182
183 // Create an audio processing object.
184 ap_ = builder->Create();
185 RTC_CHECK(ap_);
186 }
187 }
188
~AudioProcessingSimulator()189 AudioProcessingSimulator::~AudioProcessingSimulator() {
190 if (residual_echo_likelihood_graph_writer_.is_open()) {
191 WriteEchoLikelihoodGraphFileFooter(&residual_echo_likelihood_graph_writer_);
192 residual_echo_likelihood_graph_writer_.close();
193 }
194 }
195
ProcessStream(bool fixed_interface)196 void AudioProcessingSimulator::ProcessStream(bool fixed_interface) {
197 // Optionally use the fake recording device to simulate analog gain.
198 if (settings_.simulate_mic_gain) {
199 if (settings_.aec_dump_input_filename) {
200 // When the analog gain is simulated and an AEC dump is used as input, set
201 // the undo level to |aec_dump_mic_level_| to virtually restore the
202 // unmodified microphone signal level.
203 fake_recording_device_.SetUndoMicLevel(aec_dump_mic_level_);
204 }
205
206 if (fixed_interface) {
207 fake_recording_device_.SimulateAnalogGain(fwd_frame_.data);
208 } else {
209 fake_recording_device_.SimulateAnalogGain(in_buf_.get());
210 }
211
212 // Notify the current mic level to AGC.
213 ap_->set_stream_analog_level(fake_recording_device_.MicLevel());
214 } else {
215 // Notify the current mic level to AGC.
216 ap_->set_stream_analog_level(settings_.aec_dump_input_filename
217 ? aec_dump_mic_level_
218 : analog_mic_level_);
219 }
220
221 // Process the current audio frame.
222 if (fixed_interface) {
223 {
224 const auto st = ScopedTimer(&api_call_statistics_,
225 ApiCallStatistics::CallType::kCapture);
226 RTC_CHECK_EQ(
227 AudioProcessing::kNoError,
228 ap_->ProcessStream(fwd_frame_.data.data(), fwd_frame_.config,
229 fwd_frame_.config, fwd_frame_.data.data()));
230 }
231 fwd_frame_.CopyTo(out_buf_.get());
232 } else {
233 const auto st = ScopedTimer(&api_call_statistics_,
234 ApiCallStatistics::CallType::kCapture);
235 RTC_CHECK_EQ(AudioProcessing::kNoError,
236 ap_->ProcessStream(in_buf_->channels(), in_config_,
237 out_config_, out_buf_->channels()));
238 }
239
240 // Store the mic level suggested by AGC.
241 // Note that when the analog gain is simulated and an AEC dump is used as
242 // input, |analog_mic_level_| will not be used with set_stream_analog_level().
243 analog_mic_level_ = ap_->recommended_stream_analog_level();
244 if (settings_.simulate_mic_gain) {
245 fake_recording_device_.SetMicLevel(analog_mic_level_);
246 }
247 if (buffer_memory_writer_) {
248 RTC_CHECK(!buffer_file_writer_);
249 buffer_memory_writer_->Write(*out_buf_);
250 } else if (buffer_file_writer_) {
251 RTC_CHECK(!buffer_memory_writer_);
252 buffer_file_writer_->Write(*out_buf_);
253 }
254
255 if (linear_aec_output_file_writer_) {
256 bool output_available = ap_->GetLinearAecOutput(linear_aec_output_buf_);
257 RTC_CHECK(output_available);
258 RTC_CHECK_GT(linear_aec_output_buf_.size(), 0);
259 RTC_CHECK_EQ(linear_aec_output_buf_[0].size(), 160);
260 for (size_t k = 0; k < linear_aec_output_buf_[0].size(); ++k) {
261 for (size_t ch = 0; ch < linear_aec_output_buf_.size(); ++ch) {
262 RTC_CHECK_EQ(linear_aec_output_buf_[ch].size(), 160);
263 linear_aec_output_file_writer_->WriteSamples(
264 &linear_aec_output_buf_[ch][k], 1);
265 }
266 }
267 }
268
269 if (residual_echo_likelihood_graph_writer_.is_open()) {
270 auto stats = ap_->GetStatistics();
271 residual_echo_likelihood_graph_writer_
272 << stats.residual_echo_likelihood.value_or(-1.f) << ", ";
273 }
274
275 ++num_process_stream_calls_;
276 }
277
ProcessReverseStream(bool fixed_interface)278 void AudioProcessingSimulator::ProcessReverseStream(bool fixed_interface) {
279 if (fixed_interface) {
280 {
281 const auto st = ScopedTimer(&api_call_statistics_,
282 ApiCallStatistics::CallType::kRender);
283 RTC_CHECK_EQ(
284 AudioProcessing::kNoError,
285 ap_->ProcessReverseStream(rev_frame_.data.data(), rev_frame_.config,
286 rev_frame_.config, rev_frame_.data.data()));
287 }
288 rev_frame_.CopyTo(reverse_out_buf_.get());
289 } else {
290 const auto st = ScopedTimer(&api_call_statistics_,
291 ApiCallStatistics::CallType::kRender);
292 RTC_CHECK_EQ(AudioProcessing::kNoError,
293 ap_->ProcessReverseStream(
294 reverse_in_buf_->channels(), reverse_in_config_,
295 reverse_out_config_, reverse_out_buf_->channels()));
296 }
297
298 if (reverse_buffer_file_writer_) {
299 reverse_buffer_file_writer_->Write(*reverse_out_buf_);
300 }
301
302 ++num_reverse_process_stream_calls_;
303 }
304
SetupBuffersConfigsOutputs(int input_sample_rate_hz,int output_sample_rate_hz,int reverse_input_sample_rate_hz,int reverse_output_sample_rate_hz,int input_num_channels,int output_num_channels,int reverse_input_num_channels,int reverse_output_num_channels)305 void AudioProcessingSimulator::SetupBuffersConfigsOutputs(
306 int input_sample_rate_hz,
307 int output_sample_rate_hz,
308 int reverse_input_sample_rate_hz,
309 int reverse_output_sample_rate_hz,
310 int input_num_channels,
311 int output_num_channels,
312 int reverse_input_num_channels,
313 int reverse_output_num_channels) {
314 in_config_ = StreamConfig(input_sample_rate_hz, input_num_channels);
315 in_buf_.reset(new ChannelBuffer<float>(
316 rtc::CheckedDivExact(input_sample_rate_hz, kChunksPerSecond),
317 input_num_channels));
318
319 reverse_in_config_ =
320 StreamConfig(reverse_input_sample_rate_hz, reverse_input_num_channels);
321 reverse_in_buf_.reset(new ChannelBuffer<float>(
322 rtc::CheckedDivExact(reverse_input_sample_rate_hz, kChunksPerSecond),
323 reverse_input_num_channels));
324
325 out_config_ = StreamConfig(output_sample_rate_hz, output_num_channels);
326 out_buf_.reset(new ChannelBuffer<float>(
327 rtc::CheckedDivExact(output_sample_rate_hz, kChunksPerSecond),
328 output_num_channels));
329
330 reverse_out_config_ =
331 StreamConfig(reverse_output_sample_rate_hz, reverse_output_num_channels);
332 reverse_out_buf_.reset(new ChannelBuffer<float>(
333 rtc::CheckedDivExact(reverse_output_sample_rate_hz, kChunksPerSecond),
334 reverse_output_num_channels));
335
336 fwd_frame_.SetFormat(input_sample_rate_hz, input_num_channels);
337 rev_frame_.SetFormat(reverse_input_sample_rate_hz,
338 reverse_input_num_channels);
339
340 if (settings_.use_verbose_logging) {
341 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
342
343 std::cout << "Sample rates:" << std::endl;
344 std::cout << " Forward input: " << input_sample_rate_hz << std::endl;
345 std::cout << " Forward output: " << output_sample_rate_hz << std::endl;
346 std::cout << " Reverse input: " << reverse_input_sample_rate_hz
347 << std::endl;
348 std::cout << " Reverse output: " << reverse_output_sample_rate_hz
349 << std::endl;
350 std::cout << "Number of channels: " << std::endl;
351 std::cout << " Forward input: " << input_num_channels << std::endl;
352 std::cout << " Forward output: " << output_num_channels << std::endl;
353 std::cout << " Reverse input: " << reverse_input_num_channels << std::endl;
354 std::cout << " Reverse output: " << reverse_output_num_channels
355 << std::endl;
356 }
357
358 SetupOutput();
359 }
360
SetupOutput()361 void AudioProcessingSimulator::SetupOutput() {
362 if (settings_.output_filename) {
363 std::string filename;
364 if (settings_.store_intermediate_output) {
365 filename = GetIndexedOutputWavFilename(*settings_.output_filename,
366 output_reset_counter_);
367 } else {
368 filename = *settings_.output_filename;
369 }
370
371 std::unique_ptr<WavWriter> out_file(
372 new WavWriter(filename, out_config_.sample_rate_hz(),
373 static_cast<size_t>(out_config_.num_channels()),
374 settings_.wav_output_format));
375 buffer_file_writer_.reset(new ChannelBufferWavWriter(std::move(out_file)));
376 } else if (settings_.aec_dump_input_string.has_value()) {
377 buffer_memory_writer_ = std::make_unique<ChannelBufferVectorWriter>(
378 settings_.processed_capture_samples);
379 }
380
381 if (settings_.linear_aec_output_filename) {
382 std::string filename;
383 if (settings_.store_intermediate_output) {
384 filename = GetIndexedOutputWavFilename(
385 *settings_.linear_aec_output_filename, output_reset_counter_);
386 } else {
387 filename = *settings_.linear_aec_output_filename;
388 }
389
390 linear_aec_output_file_writer_.reset(
391 new WavWriter(filename, 16000, out_config_.num_channels(),
392 settings_.wav_output_format));
393
394 linear_aec_output_buf_.resize(out_config_.num_channels());
395 }
396
397 if (settings_.reverse_output_filename) {
398 std::string filename;
399 if (settings_.store_intermediate_output) {
400 filename = GetIndexedOutputWavFilename(*settings_.reverse_output_filename,
401 output_reset_counter_);
402 } else {
403 filename = *settings_.reverse_output_filename;
404 }
405
406 std::unique_ptr<WavWriter> reverse_out_file(
407 new WavWriter(filename, reverse_out_config_.sample_rate_hz(),
408 static_cast<size_t>(reverse_out_config_.num_channels()),
409 settings_.wav_output_format));
410 reverse_buffer_file_writer_.reset(
411 new ChannelBufferWavWriter(std::move(reverse_out_file)));
412 }
413
414 ++output_reset_counter_;
415 }
416
DetachAecDump()417 void AudioProcessingSimulator::DetachAecDump() {
418 if (settings_.aec_dump_output_filename) {
419 ap_->DetachAecDump();
420 }
421 }
422
ConfigureAudioProcessor()423 void AudioProcessingSimulator::ConfigureAudioProcessor() {
424 AudioProcessing::Config apm_config;
425 if (settings_.use_ts) {
426 apm_config.transient_suppression.enabled = *settings_.use_ts;
427 }
428 if (settings_.multi_channel_render) {
429 apm_config.pipeline.multi_channel_render = *settings_.multi_channel_render;
430 }
431
432 if (settings_.multi_channel_capture) {
433 apm_config.pipeline.multi_channel_capture =
434 *settings_.multi_channel_capture;
435 }
436
437 if (settings_.use_agc2) {
438 apm_config.gain_controller2.enabled = *settings_.use_agc2;
439 if (settings_.agc2_fixed_gain_db) {
440 apm_config.gain_controller2.fixed_digital.gain_db =
441 *settings_.agc2_fixed_gain_db;
442 }
443 if (settings_.agc2_use_adaptive_gain) {
444 apm_config.gain_controller2.adaptive_digital.enabled =
445 *settings_.agc2_use_adaptive_gain;
446 apm_config.gain_controller2.adaptive_digital.level_estimator =
447 settings_.agc2_adaptive_level_estimator;
448 }
449 }
450 if (settings_.use_pre_amplifier) {
451 apm_config.pre_amplifier.enabled = *settings_.use_pre_amplifier;
452 if (settings_.pre_amplifier_gain_factor) {
453 apm_config.pre_amplifier.fixed_gain_factor =
454 *settings_.pre_amplifier_gain_factor;
455 }
456 }
457
458 const bool use_aec = settings_.use_aec && *settings_.use_aec;
459 const bool use_aecm = settings_.use_aecm && *settings_.use_aecm;
460 if (use_aec || use_aecm) {
461 apm_config.echo_canceller.enabled = true;
462 apm_config.echo_canceller.mobile_mode = use_aecm;
463 }
464 apm_config.echo_canceller.export_linear_aec_output =
465 !!settings_.linear_aec_output_filename;
466
467 if (settings_.use_hpf) {
468 apm_config.high_pass_filter.enabled = *settings_.use_hpf;
469 }
470
471 if (settings_.use_le) {
472 apm_config.level_estimation.enabled = *settings_.use_le;
473 }
474
475 if (settings_.use_vad) {
476 apm_config.voice_detection.enabled = *settings_.use_vad;
477 }
478
479 if (settings_.use_agc) {
480 apm_config.gain_controller1.enabled = *settings_.use_agc;
481 }
482 if (settings_.agc_mode) {
483 apm_config.gain_controller1.mode =
484 static_cast<webrtc::AudioProcessing::Config::GainController1::Mode>(
485 *settings_.agc_mode);
486 }
487 if (settings_.use_agc_limiter) {
488 apm_config.gain_controller1.enable_limiter = *settings_.use_agc_limiter;
489 }
490 if (settings_.agc_target_level) {
491 apm_config.gain_controller1.target_level_dbfs = *settings_.agc_target_level;
492 }
493 if (settings_.agc_compression_gain) {
494 apm_config.gain_controller1.compression_gain_db =
495 *settings_.agc_compression_gain;
496 }
497 if (settings_.use_analog_agc) {
498 apm_config.gain_controller1.analog_gain_controller.enabled =
499 *settings_.use_analog_agc;
500 }
501 if (settings_.use_analog_agc_agc2_level_estimator) {
502 apm_config.gain_controller1.analog_gain_controller
503 .enable_agc2_level_estimator =
504 *settings_.use_analog_agc_agc2_level_estimator;
505 }
506 if (settings_.analog_agc_disable_digital_adaptive) {
507 apm_config.gain_controller1.analog_gain_controller.enable_digital_adaptive =
508 *settings_.analog_agc_disable_digital_adaptive;
509 }
510
511 if (settings_.use_ed) {
512 apm_config.residual_echo_detector.enabled = *settings_.use_ed;
513 }
514
515 if (settings_.maximum_internal_processing_rate) {
516 apm_config.pipeline.maximum_internal_processing_rate =
517 *settings_.maximum_internal_processing_rate;
518 }
519
520 if (settings_.use_ns) {
521 apm_config.noise_suppression.enabled = *settings_.use_ns;
522 }
523 if (settings_.ns_level) {
524 const int level = *settings_.ns_level;
525 RTC_CHECK_GE(level, 0);
526 RTC_CHECK_LE(level, 3);
527 apm_config.noise_suppression.level =
528 static_cast<AudioProcessing::Config::NoiseSuppression::Level>(level);
529 }
530 if (settings_.ns_analysis_on_linear_aec_output) {
531 apm_config.noise_suppression.analyze_linear_aec_output_when_available =
532 *settings_.ns_analysis_on_linear_aec_output;
533 }
534
535 ap_->ApplyConfig(apm_config);
536
537 if (settings_.use_ts) {
538 ap_->set_stream_key_pressed(*settings_.use_ts);
539 }
540
541 if (settings_.aec_dump_output_filename) {
542 ap_->AttachAecDump(AecDumpFactory::Create(
543 *settings_.aec_dump_output_filename, -1, &worker_queue_));
544 }
545 }
546
547 } // namespace test
548 } // namespace webrtc
549