1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/gain_control_impl.h"
12
13 #include <cstdint>
14
15 #include "absl/types/optional.h"
16 #include "modules/audio_processing/agc/legacy/gain_control.h"
17 #include "modules/audio_processing/audio_buffer.h"
18 #include "modules/audio_processing/include/audio_processing.h"
19 #include "modules/audio_processing/logging/apm_data_dumper.h"
20 #include "rtc_base/checks.h"
21 #include "rtc_base/logging.h"
22 #include "system_wrappers/include/field_trial.h"
23
24 namespace webrtc {
25
26 typedef void Handle;
27
28 namespace {
MapSetting(GainControl::Mode mode)29 int16_t MapSetting(GainControl::Mode mode) {
30 switch (mode) {
31 case GainControl::kAdaptiveAnalog:
32 return kAgcModeAdaptiveAnalog;
33 case GainControl::kAdaptiveDigital:
34 return kAgcModeAdaptiveDigital;
35 case GainControl::kFixedDigital:
36 return kAgcModeFixedDigital;
37 }
38 RTC_NOTREACHED();
39 return -1;
40 }
41
42 // Checks whether the legacy digital gain application should be used.
UseLegacyDigitalGainApplier()43 bool UseLegacyDigitalGainApplier() {
44 return field_trial::IsEnabled("WebRTC-UseLegacyDigitalGainApplier");
45 }
46
47 // Floating point variant of WebRtcAgc_Process.
ApplyDigitalGain(const int32_t gains[11],size_t num_bands,float * const * out)48 void ApplyDigitalGain(const int32_t gains[11],
49 size_t num_bands,
50 float* const* out) {
51 constexpr float kScaling = 1.f / 65536.f;
52 constexpr int kNumSubSections = 16;
53 constexpr float kOneByNumSubSections = 1.f / kNumSubSections;
54
55 float gains_scaled[11];
56 for (int k = 0; k < 11; ++k) {
57 gains_scaled[k] = gains[k] * kScaling;
58 }
59
60 for (size_t b = 0; b < num_bands; ++b) {
61 float* out_band = out[b];
62 for (int k = 0, sample = 0; k < 10; ++k) {
63 const float delta =
64 (gains_scaled[k + 1] - gains_scaled[k]) * kOneByNumSubSections;
65 float gain = gains_scaled[k];
66 for (int n = 0; n < kNumSubSections; ++n, ++sample) {
67 RTC_DCHECK_EQ(k * kNumSubSections + n, sample);
68 out_band[sample] *= gain;
69 out_band[sample] =
70 std::min(32767.f, std::max(-32768.f, out_band[sample]));
71 gain += delta;
72 }
73 }
74 }
75 }
76
77 } // namespace
78
79 struct GainControlImpl::MonoAgcState {
MonoAgcStatewebrtc::GainControlImpl::MonoAgcState80 MonoAgcState() {
81 state = WebRtcAgc_Create();
82 RTC_CHECK(state);
83 }
84
~MonoAgcStatewebrtc::GainControlImpl::MonoAgcState85 ~MonoAgcState() {
86 RTC_DCHECK(state);
87 WebRtcAgc_Free(state);
88 }
89
90 MonoAgcState(const MonoAgcState&) = delete;
91 MonoAgcState& operator=(const MonoAgcState&) = delete;
92 int32_t gains[11];
93 Handle* state;
94 };
95
96 int GainControlImpl::instance_counter_ = 0;
97
GainControlImpl()98 GainControlImpl::GainControlImpl()
99 : data_dumper_(new ApmDataDumper(instance_counter_)),
100 use_legacy_gain_applier_(UseLegacyDigitalGainApplier()),
101 mode_(kAdaptiveAnalog),
102 minimum_capture_level_(0),
103 maximum_capture_level_(255),
104 limiter_enabled_(true),
105 target_level_dbfs_(3),
106 compression_gain_db_(9),
107 analog_capture_level_(0),
108 was_analog_level_set_(false),
109 stream_is_saturated_(false) {}
110
111 GainControlImpl::~GainControlImpl() = default;
112
ProcessRenderAudio(rtc::ArrayView<const int16_t> packed_render_audio)113 void GainControlImpl::ProcessRenderAudio(
114 rtc::ArrayView<const int16_t> packed_render_audio) {
115 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
116 WebRtcAgc_AddFarend(mono_agcs_[ch]->state, packed_render_audio.data(),
117 packed_render_audio.size());
118 }
119 }
120
PackRenderAudioBuffer(const AudioBuffer & audio,std::vector<int16_t> * packed_buffer)121 void GainControlImpl::PackRenderAudioBuffer(
122 const AudioBuffer& audio,
123 std::vector<int16_t>* packed_buffer) {
124 RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength, audio.num_frames_per_band());
125 std::array<int16_t, AudioBuffer::kMaxSplitFrameLength>
126 mixed_16_kHz_render_data;
127 rtc::ArrayView<const int16_t> mixed_16_kHz_render(
128 mixed_16_kHz_render_data.data(), audio.num_frames_per_band());
129 if (audio.num_channels() == 1) {
130 FloatS16ToS16(audio.split_bands_const(0)[kBand0To8kHz],
131 audio.num_frames_per_band(), mixed_16_kHz_render_data.data());
132 } else {
133 const int num_channels = static_cast<int>(audio.num_channels());
134 for (size_t i = 0; i < audio.num_frames_per_band(); ++i) {
135 int32_t sum = 0;
136 for (int ch = 0; ch < num_channels; ++ch) {
137 sum += FloatS16ToS16(audio.split_channels_const(kBand0To8kHz)[ch][i]);
138 }
139 mixed_16_kHz_render_data[i] = sum / num_channels;
140 }
141 }
142
143 packed_buffer->clear();
144 packed_buffer->insert(
145 packed_buffer->end(), mixed_16_kHz_render.data(),
146 (mixed_16_kHz_render.data() + audio.num_frames_per_band()));
147 }
148
AnalyzeCaptureAudio(const AudioBuffer & audio)149 int GainControlImpl::AnalyzeCaptureAudio(const AudioBuffer& audio) {
150 RTC_DCHECK(num_proc_channels_);
151 RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength, audio.num_frames_per_band());
152 RTC_DCHECK_EQ(audio.num_channels(), *num_proc_channels_);
153 RTC_DCHECK_LE(*num_proc_channels_, mono_agcs_.size());
154
155 int16_t split_band_data[AudioBuffer::kMaxNumBands]
156 [AudioBuffer::kMaxSplitFrameLength];
157 int16_t* split_bands[AudioBuffer::kMaxNumBands] = {
158 split_band_data[0], split_band_data[1], split_band_data[2]};
159
160 if (mode_ == kAdaptiveAnalog) {
161 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
162 capture_levels_[ch] = analog_capture_level_;
163
164 audio.ExportSplitChannelData(ch, split_bands);
165
166 int err =
167 WebRtcAgc_AddMic(mono_agcs_[ch]->state, split_bands,
168 audio.num_bands(), audio.num_frames_per_band());
169
170 if (err != AudioProcessing::kNoError) {
171 return AudioProcessing::kUnspecifiedError;
172 }
173 }
174 } else if (mode_ == kAdaptiveDigital) {
175 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
176 int32_t capture_level_out = 0;
177
178 audio.ExportSplitChannelData(ch, split_bands);
179
180 int err =
181 WebRtcAgc_VirtualMic(mono_agcs_[ch]->state, split_bands,
182 audio.num_bands(), audio.num_frames_per_band(),
183 analog_capture_level_, &capture_level_out);
184
185 capture_levels_[ch] = capture_level_out;
186
187 if (err != AudioProcessing::kNoError) {
188 return AudioProcessing::kUnspecifiedError;
189 }
190 }
191 }
192
193 return AudioProcessing::kNoError;
194 }
195
ProcessCaptureAudio(AudioBuffer * audio,bool stream_has_echo)196 int GainControlImpl::ProcessCaptureAudio(AudioBuffer* audio,
197 bool stream_has_echo) {
198 if (mode_ == kAdaptiveAnalog && !was_analog_level_set_) {
199 return AudioProcessing::kStreamParameterNotSetError;
200 }
201
202 RTC_DCHECK(num_proc_channels_);
203 RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength,
204 audio->num_frames_per_band());
205 RTC_DCHECK_EQ(audio->num_channels(), *num_proc_channels_);
206
207 stream_is_saturated_ = false;
208 bool error_reported = false;
209 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
210 int16_t split_band_data[AudioBuffer::kMaxNumBands]
211 [AudioBuffer::kMaxSplitFrameLength];
212 int16_t* split_bands[AudioBuffer::kMaxNumBands] = {
213 split_band_data[0], split_band_data[1], split_band_data[2]};
214 audio->ExportSplitChannelData(ch, split_bands);
215
216 // The call to stream_has_echo() is ok from a deadlock perspective
217 // as the capture lock is allready held.
218 int32_t new_capture_level = 0;
219 uint8_t saturation_warning = 0;
220 int err_analyze = WebRtcAgc_Analyze(
221 mono_agcs_[ch]->state, split_bands, audio->num_bands(),
222 audio->num_frames_per_band(), capture_levels_[ch], &new_capture_level,
223 stream_has_echo, &saturation_warning, mono_agcs_[ch]->gains);
224 capture_levels_[ch] = new_capture_level;
225
226 error_reported = error_reported || err_analyze != AudioProcessing::kNoError;
227
228 stream_is_saturated_ = stream_is_saturated_ || saturation_warning == 1;
229 }
230
231 // Choose the minimun gain for application
232 size_t index_to_apply = 0;
233 for (size_t ch = 1; ch < mono_agcs_.size(); ++ch) {
234 if (mono_agcs_[index_to_apply]->gains[10] < mono_agcs_[ch]->gains[10]) {
235 index_to_apply = ch;
236 }
237 }
238
239 if (use_legacy_gain_applier_) {
240 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
241 int16_t split_band_data[AudioBuffer::kMaxNumBands]
242 [AudioBuffer::kMaxSplitFrameLength];
243 int16_t* split_bands[AudioBuffer::kMaxNumBands] = {
244 split_band_data[0], split_band_data[1], split_band_data[2]};
245 audio->ExportSplitChannelData(ch, split_bands);
246
247 int err_process = WebRtcAgc_Process(
248 mono_agcs_[ch]->state, mono_agcs_[index_to_apply]->gains, split_bands,
249 audio->num_bands(), split_bands);
250 RTC_DCHECK_EQ(err_process, 0);
251
252 audio->ImportSplitChannelData(ch, split_bands);
253 }
254 } else {
255 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
256 ApplyDigitalGain(mono_agcs_[index_to_apply]->gains, audio->num_bands(),
257 audio->split_bands(ch));
258 }
259 }
260
261 RTC_DCHECK_LT(0ul, *num_proc_channels_);
262 if (mode_ == kAdaptiveAnalog) {
263 // Take the analog level to be the minimum accross all channels.
264 analog_capture_level_ = capture_levels_[0];
265 for (size_t ch = 1; ch < mono_agcs_.size(); ++ch) {
266 analog_capture_level_ =
267 std::min(analog_capture_level_, capture_levels_[ch]);
268 }
269 }
270
271 if (error_reported) {
272 return AudioProcessing::kUnspecifiedError;
273 }
274
275 was_analog_level_set_ = false;
276
277 return AudioProcessing::kNoError;
278 }
279
280
281 // TODO(ajm): ensure this is called under kAdaptiveAnalog.
set_stream_analog_level(int level)282 int GainControlImpl::set_stream_analog_level(int level) {
283 data_dumper_->DumpRaw("gain_control_set_stream_analog_level", 1, &level);
284
285 was_analog_level_set_ = true;
286 if (level < minimum_capture_level_ || level > maximum_capture_level_) {
287 return AudioProcessing::kBadParameterError;
288 }
289 analog_capture_level_ = level;
290
291 return AudioProcessing::kNoError;
292 }
293
stream_analog_level() const294 int GainControlImpl::stream_analog_level() const {
295 data_dumper_->DumpRaw("gain_control_stream_analog_level", 1,
296 &analog_capture_level_);
297 return analog_capture_level_;
298 }
299
set_mode(Mode mode)300 int GainControlImpl::set_mode(Mode mode) {
301 if (MapSetting(mode) == -1) {
302 return AudioProcessing::kBadParameterError;
303 }
304
305 mode_ = mode;
306 RTC_DCHECK(num_proc_channels_);
307 RTC_DCHECK(sample_rate_hz_);
308 Initialize(*num_proc_channels_, *sample_rate_hz_);
309 return AudioProcessing::kNoError;
310 }
311
312
set_analog_level_limits(int minimum,int maximum)313 int GainControlImpl::set_analog_level_limits(int minimum, int maximum) {
314 if (minimum < 0 || maximum > 65535 || maximum < minimum) {
315 return AudioProcessing::kBadParameterError;
316 }
317
318 minimum_capture_level_ = minimum;
319 maximum_capture_level_ = maximum;
320
321 RTC_DCHECK(num_proc_channels_);
322 RTC_DCHECK(sample_rate_hz_);
323 Initialize(*num_proc_channels_, *sample_rate_hz_);
324 return AudioProcessing::kNoError;
325 }
326
327
set_target_level_dbfs(int level)328 int GainControlImpl::set_target_level_dbfs(int level) {
329 if (level > 31 || level < 0) {
330 return AudioProcessing::kBadParameterError;
331 }
332 target_level_dbfs_ = level;
333 return Configure();
334 }
335
set_compression_gain_db(int gain)336 int GainControlImpl::set_compression_gain_db(int gain) {
337 if (gain < 0 || gain > 90) {
338 RTC_LOG(LS_ERROR) << "set_compression_gain_db(" << gain << ") failed.";
339 return AudioProcessing::kBadParameterError;
340 }
341 compression_gain_db_ = gain;
342 return Configure();
343 }
344
enable_limiter(bool enable)345 int GainControlImpl::enable_limiter(bool enable) {
346 limiter_enabled_ = enable;
347 return Configure();
348 }
349
Initialize(size_t num_proc_channels,int sample_rate_hz)350 void GainControlImpl::Initialize(size_t num_proc_channels, int sample_rate_hz) {
351 data_dumper_->InitiateNewSetOfRecordings();
352
353 RTC_DCHECK(sample_rate_hz == 16000 || sample_rate_hz == 32000 ||
354 sample_rate_hz == 48000);
355
356 num_proc_channels_ = num_proc_channels;
357 sample_rate_hz_ = sample_rate_hz;
358
359 mono_agcs_.resize(*num_proc_channels_);
360 capture_levels_.resize(*num_proc_channels_);
361 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
362 if (!mono_agcs_[ch]) {
363 mono_agcs_[ch].reset(new MonoAgcState());
364 }
365
366 int error = WebRtcAgc_Init(mono_agcs_[ch]->state, minimum_capture_level_,
367 maximum_capture_level_, MapSetting(mode_),
368 *sample_rate_hz_);
369 RTC_DCHECK_EQ(error, 0);
370 capture_levels_[ch] = analog_capture_level_;
371 }
372
373 Configure();
374 }
375
Configure()376 int GainControlImpl::Configure() {
377 WebRtcAgcConfig config;
378 // TODO(ajm): Flip the sign here (since AGC expects a positive value) if we
379 // change the interface.
380 // RTC_DCHECK_LE(target_level_dbfs_, 0);
381 // config.targetLevelDbfs = static_cast<int16_t>(-target_level_dbfs_);
382 config.targetLevelDbfs = static_cast<int16_t>(target_level_dbfs_);
383 config.compressionGaindB = static_cast<int16_t>(compression_gain_db_);
384 config.limiterEnable = limiter_enabled_;
385
386 int error = AudioProcessing::kNoError;
387 for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) {
388 int error_ch = WebRtcAgc_set_config(mono_agcs_[ch]->state, config);
389 if (error_ch != AudioProcessing::kNoError) {
390 error = error_ch;
391 }
392 }
393 return error;
394 }
395 } // namespace webrtc
396