1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
12 
13 #include <assert.h>
14 #include <string.h>
15 
16 #include "webrtc/modules/audio_processing/aecm/echo_control_mobile.h"
17 #include "webrtc/modules/audio_processing/audio_buffer.h"
18 #include "webrtc/system_wrappers/include/logging.h"
19 
20 namespace webrtc {
21 
22 typedef void Handle;
23 
24 namespace {
MapSetting(EchoControlMobile::RoutingMode mode)25 int16_t MapSetting(EchoControlMobile::RoutingMode mode) {
26   switch (mode) {
27     case EchoControlMobile::kQuietEarpieceOrHeadset:
28       return 0;
29     case EchoControlMobile::kEarpiece:
30       return 1;
31     case EchoControlMobile::kLoudEarpiece:
32       return 2;
33     case EchoControlMobile::kSpeakerphone:
34       return 3;
35     case EchoControlMobile::kLoudSpeakerphone:
36       return 4;
37   }
38   assert(false);
39   return -1;
40 }
41 
MapError(int err)42 AudioProcessing::Error MapError(int err) {
43   switch (err) {
44     case AECM_UNSUPPORTED_FUNCTION_ERROR:
45       return AudioProcessing::kUnsupportedFunctionError;
46     case AECM_NULL_POINTER_ERROR:
47       return AudioProcessing::kNullPointerError;
48     case AECM_BAD_PARAMETER_ERROR:
49       return AudioProcessing::kBadParameterError;
50     case AECM_BAD_PARAMETER_WARNING:
51       return AudioProcessing::kBadStreamParameterWarning;
52     default:
53       // AECM_UNSPECIFIED_ERROR
54       // AECM_UNINITIALIZED_ERROR
55       return AudioProcessing::kUnspecifiedError;
56   }
57 }
58 // Maximum length that a frame of samples can have.
59 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
60 // Maximum number of frames to buffer in the render queue.
61 // TODO(peah): Decrease this once we properly handle hugely unbalanced
62 // reverse and forward call numbers.
63 static const size_t kMaxNumFramesToBuffer = 100;
64 }  // namespace
65 
echo_path_size_bytes()66 size_t EchoControlMobile::echo_path_size_bytes() {
67     return WebRtcAecm_echo_path_size_bytes();
68 }
69 
EchoControlMobileImpl(const AudioProcessing * apm,rtc::CriticalSection * crit_render,rtc::CriticalSection * crit_capture)70 EchoControlMobileImpl::EchoControlMobileImpl(const AudioProcessing* apm,
71                                              rtc::CriticalSection* crit_render,
72                                              rtc::CriticalSection* crit_capture)
73     : ProcessingComponent(),
74       apm_(apm),
75       crit_render_(crit_render),
76       crit_capture_(crit_capture),
77       routing_mode_(kSpeakerphone),
78       comfort_noise_enabled_(true),
79       external_echo_path_(NULL),
80       render_queue_element_max_size_(0) {
81   RTC_DCHECK(apm);
82   RTC_DCHECK(crit_render);
83   RTC_DCHECK(crit_capture);
84 }
85 
~EchoControlMobileImpl()86 EchoControlMobileImpl::~EchoControlMobileImpl() {
87     if (external_echo_path_ != NULL) {
88       delete [] external_echo_path_;
89       external_echo_path_ = NULL;
90     }
91 }
92 
ProcessRenderAudio(const AudioBuffer * audio)93 int EchoControlMobileImpl::ProcessRenderAudio(const AudioBuffer* audio) {
94   rtc::CritScope cs_render(crit_render_);
95 
96   if (!is_component_enabled()) {
97     return AudioProcessing::kNoError;
98   }
99 
100   assert(audio->num_frames_per_band() <= 160);
101   assert(audio->num_channels() == apm_->num_reverse_channels());
102 
103   int err = AudioProcessing::kNoError;
104   // The ordering convention must be followed to pass to the correct AECM.
105   size_t handle_index = 0;
106   render_queue_buffer_.clear();
107   for (size_t i = 0; i < apm_->num_output_channels(); i++) {
108     for (size_t j = 0; j < audio->num_channels(); j++) {
109       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
110       err = WebRtcAecm_GetBufferFarendError(
111           my_handle, audio->split_bands_const(j)[kBand0To8kHz],
112           audio->num_frames_per_band());
113 
114       if (err != AudioProcessing::kNoError)
115         return MapError(err);  // TODO(ajm): warning possible?);
116 
117       // Buffer the samples in the render queue.
118       render_queue_buffer_.insert(render_queue_buffer_.end(),
119                                   audio->split_bands_const(j)[kBand0To8kHz],
120                                   (audio->split_bands_const(j)[kBand0To8kHz] +
121                                    audio->num_frames_per_band()));
122 
123       handle_index++;
124     }
125   }
126 
127   // Insert the samples into the queue.
128   if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
129     // The data queue is full and needs to be emptied.
130     ReadQueuedRenderData();
131 
132     // Retry the insert (should always work).
133     RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
134   }
135 
136   return AudioProcessing::kNoError;
137 }
138 
139 // Read chunks of data that were received and queued on the render side from
140 // a queue. All the data chunks are buffered into the farend signal of the AEC.
ReadQueuedRenderData()141 void EchoControlMobileImpl::ReadQueuedRenderData() {
142   rtc::CritScope cs_capture(crit_capture_);
143 
144   if (!is_component_enabled()) {
145     return;
146   }
147 
148   while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
149     size_t handle_index = 0;
150     size_t buffer_index = 0;
151     const size_t num_frames_per_band =
152         capture_queue_buffer_.size() /
153         (apm_->num_output_channels() * apm_->num_reverse_channels());
154     for (size_t i = 0; i < apm_->num_output_channels(); i++) {
155       for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
156         Handle* my_handle = static_cast<Handle*>(handle(handle_index));
157         WebRtcAecm_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index],
158                                 num_frames_per_band);
159 
160         buffer_index += num_frames_per_band;
161         handle_index++;
162       }
163     }
164   }
165 }
166 
ProcessCaptureAudio(AudioBuffer * audio)167 int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) {
168   rtc::CritScope cs_capture(crit_capture_);
169 
170   if (!is_component_enabled()) {
171     return AudioProcessing::kNoError;
172   }
173 
174   if (!apm_->was_stream_delay_set()) {
175     return AudioProcessing::kStreamParameterNotSetError;
176   }
177 
178   assert(audio->num_frames_per_band() <= 160);
179   assert(audio->num_channels() == apm_->num_output_channels());
180 
181   int err = AudioProcessing::kNoError;
182 
183   // The ordering convention must be followed to pass to the correct AECM.
184   size_t handle_index = 0;
185   for (size_t i = 0; i < audio->num_channels(); i++) {
186     // TODO(ajm): improve how this works, possibly inside AECM.
187     //            This is kind of hacked up.
188     const int16_t* noisy = audio->low_pass_reference(i);
189     const int16_t* clean = audio->split_bands_const(i)[kBand0To8kHz];
190     if (noisy == NULL) {
191       noisy = clean;
192       clean = NULL;
193     }
194     for (size_t j = 0; j < apm_->num_reverse_channels(); j++) {
195       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
196       err = WebRtcAecm_Process(
197           my_handle,
198           noisy,
199           clean,
200           audio->split_bands(i)[kBand0To8kHz],
201           audio->num_frames_per_band(),
202           apm_->stream_delay_ms());
203 
204       if (err != AudioProcessing::kNoError)
205         return MapError(err);
206 
207       handle_index++;
208     }
209   }
210 
211   return AudioProcessing::kNoError;
212 }
213 
Enable(bool enable)214 int EchoControlMobileImpl::Enable(bool enable) {
215   // Ensure AEC and AECM are not both enabled.
216   rtc::CritScope cs_render(crit_render_);
217   rtc::CritScope cs_capture(crit_capture_);
218   // The is_enabled call is safe from a deadlock perspective
219   // as both locks are allready held in the correct order.
220   if (enable && apm_->echo_cancellation()->is_enabled()) {
221     return AudioProcessing::kBadParameterError;
222   }
223 
224   return EnableComponent(enable);
225 }
226 
is_enabled() const227 bool EchoControlMobileImpl::is_enabled() const {
228   rtc::CritScope cs(crit_capture_);
229   return is_component_enabled();
230 }
231 
set_routing_mode(RoutingMode mode)232 int EchoControlMobileImpl::set_routing_mode(RoutingMode mode) {
233   if (MapSetting(mode) == -1) {
234     return AudioProcessing::kBadParameterError;
235   }
236 
237   {
238     rtc::CritScope cs(crit_capture_);
239     routing_mode_ = mode;
240   }
241   return Configure();
242 }
243 
routing_mode() const244 EchoControlMobile::RoutingMode EchoControlMobileImpl::routing_mode()
245     const {
246   rtc::CritScope cs(crit_capture_);
247   return routing_mode_;
248 }
249 
enable_comfort_noise(bool enable)250 int EchoControlMobileImpl::enable_comfort_noise(bool enable) {
251   {
252     rtc::CritScope cs(crit_capture_);
253     comfort_noise_enabled_ = enable;
254   }
255   return Configure();
256 }
257 
is_comfort_noise_enabled() const258 bool EchoControlMobileImpl::is_comfort_noise_enabled() const {
259   rtc::CritScope cs(crit_capture_);
260   return comfort_noise_enabled_;
261 }
262 
SetEchoPath(const void * echo_path,size_t size_bytes)263 int EchoControlMobileImpl::SetEchoPath(const void* echo_path,
264                                        size_t size_bytes) {
265   {
266     rtc::CritScope cs_render(crit_render_);
267     rtc::CritScope cs_capture(crit_capture_);
268     if (echo_path == NULL) {
269       return AudioProcessing::kNullPointerError;
270     }
271     if (size_bytes != echo_path_size_bytes()) {
272       // Size mismatch
273       return AudioProcessing::kBadParameterError;
274     }
275 
276     if (external_echo_path_ == NULL) {
277       external_echo_path_ = new unsigned char[size_bytes];
278     }
279     memcpy(external_echo_path_, echo_path, size_bytes);
280   }
281 
282   return Initialize();
283 }
284 
GetEchoPath(void * echo_path,size_t size_bytes) const285 int EchoControlMobileImpl::GetEchoPath(void* echo_path,
286                                        size_t size_bytes) const {
287   rtc::CritScope cs(crit_capture_);
288   if (echo_path == NULL) {
289     return AudioProcessing::kNullPointerError;
290   }
291   if (size_bytes != echo_path_size_bytes()) {
292     // Size mismatch
293     return AudioProcessing::kBadParameterError;
294   }
295   if (!is_component_enabled()) {
296     return AudioProcessing::kNotEnabledError;
297   }
298 
299   // Get the echo path from the first channel
300   Handle* my_handle = static_cast<Handle*>(handle(0));
301   int32_t err = WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes);
302   if (err != 0)
303     return MapError(err);
304 
305   return AudioProcessing::kNoError;
306 }
307 
Initialize()308 int EchoControlMobileImpl::Initialize() {
309   {
310     rtc::CritScope cs_capture(crit_capture_);
311     if (!is_component_enabled()) {
312       return AudioProcessing::kNoError;
313     }
314   }
315 
316   if (apm_->proc_sample_rate_hz() > AudioProcessing::kSampleRate16kHz) {
317     LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates";
318     return AudioProcessing::kBadSampleRateError;
319   }
320 
321   int err = ProcessingComponent::Initialize();
322   if (err != AudioProcessing::kNoError) {
323     return err;
324   }
325 
326   AllocateRenderQueue();
327 
328   return AudioProcessing::kNoError;
329 }
330 
AllocateRenderQueue()331 void EchoControlMobileImpl::AllocateRenderQueue() {
332   const size_t new_render_queue_element_max_size = std::max<size_t>(
333       static_cast<size_t>(1),
334       kMaxAllowedValuesOfSamplesPerFrame * num_handles_required());
335 
336   rtc::CritScope cs_render(crit_render_);
337   rtc::CritScope cs_capture(crit_capture_);
338 
339   // Reallocate the queue if the queue item size is too small to fit the
340   // data to put in the queue.
341   if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
342     render_queue_element_max_size_ = new_render_queue_element_max_size;
343 
344     std::vector<int16_t> template_queue_element(render_queue_element_max_size_);
345 
346     render_signal_queue_.reset(
347         new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
348             kMaxNumFramesToBuffer, template_queue_element,
349             RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_)));
350 
351     render_queue_buffer_.resize(render_queue_element_max_size_);
352     capture_queue_buffer_.resize(render_queue_element_max_size_);
353   } else {
354     render_signal_queue_->Clear();
355   }
356 }
357 
CreateHandle() const358 void* EchoControlMobileImpl::CreateHandle() const {
359   return WebRtcAecm_Create();
360 }
361 
DestroyHandle(void * handle) const362 void EchoControlMobileImpl::DestroyHandle(void* handle) const {
363   // This method is only called in a non-concurrent manner during APM
364   // destruction.
365   WebRtcAecm_Free(static_cast<Handle*>(handle));
366 }
367 
InitializeHandle(void * handle) const368 int EchoControlMobileImpl::InitializeHandle(void* handle) const {
369   rtc::CritScope cs_render(crit_render_);
370   rtc::CritScope cs_capture(crit_capture_);
371   assert(handle != NULL);
372   Handle* my_handle = static_cast<Handle*>(handle);
373   if (WebRtcAecm_Init(my_handle, apm_->proc_sample_rate_hz()) != 0) {
374     return GetHandleError(my_handle);
375   }
376   if (external_echo_path_ != NULL) {
377     if (WebRtcAecm_InitEchoPath(my_handle,
378                                 external_echo_path_,
379                                 echo_path_size_bytes()) != 0) {
380       return GetHandleError(my_handle);
381     }
382   }
383 
384   return AudioProcessing::kNoError;
385 }
386 
ConfigureHandle(void * handle) const387 int EchoControlMobileImpl::ConfigureHandle(void* handle) const {
388   rtc::CritScope cs_render(crit_render_);
389   rtc::CritScope cs_capture(crit_capture_);
390   AecmConfig config;
391   config.cngMode = comfort_noise_enabled_;
392   config.echoMode = MapSetting(routing_mode_);
393 
394   return WebRtcAecm_set_config(static_cast<Handle*>(handle), config);
395 }
396 
num_handles_required() const397 size_t EchoControlMobileImpl::num_handles_required() const {
398   // Not locked as it only relies on APM public API which is threadsafe.
399   return apm_->num_output_channels() * apm_->num_reverse_channels();
400 }
401 
GetHandleError(void * handle) const402 int EchoControlMobileImpl::GetHandleError(void* handle) const {
403   // Not locked as it does not rely on anything in the state.
404   assert(handle != NULL);
405   return AudioProcessing::kUnspecifiedError;
406 }
407 }  // namespace webrtc
408