1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 /*
12  * Contains the API functions for the AEC.
13  */
14 #include "webrtc/modules/audio_processing/aec/echo_cancellation.h"
15 
16 #include <math.h>
17 #ifdef WEBRTC_AEC_DEBUG_DUMP
18 #include <stdio.h>
19 #endif
20 #include <stdlib.h>
21 #include <string.h>
22 
23 #include "webrtc/common_audio/ring_buffer.h"
24 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
25 #include "webrtc/modules/audio_processing/aec/aec_core.h"
26 #include "webrtc/modules/audio_processing/aec/aec_resampler.h"
27 #include "webrtc/modules/audio_processing/aec/echo_cancellation_internal.h"
28 #include "webrtc/typedefs.h"
29 
30 // Measured delays [ms]
31 // Device                Chrome  GTP
32 // MacBook Air           10
33 // MacBook Retina        10      100
34 // MacPro                30?
35 //
36 // Win7 Desktop          70      80?
37 // Win7 T430s            110
38 // Win8 T420s            70
39 //
40 // Daisy                 50
41 // Pixel (w/ preproc?)           240
42 // Pixel (w/o preproc?)  110     110
43 
44 // The extended filter mode gives us the flexibility to ignore the system's
45 // reported delays. We do this for platforms which we believe provide results
46 // which are incompatible with the AEC's expectations. Based on measurements
47 // (some provided above) we set a conservative (i.e. lower than measured)
48 // fixed delay.
49 //
50 // WEBRTC_UNTRUSTED_DELAY will only have an impact when |extended_filter_mode|
51 // is enabled. See the note along with |DelayCorrection| in
52 // echo_cancellation_impl.h for more details on the mode.
53 //
54 // Justification:
55 // Chromium/Mac: Here, the true latency is so low (~10-20 ms), that it plays
56 // havoc with the AEC's buffering. To avoid this, we set a fixed delay of 20 ms
57 // and then compensate by rewinding by 10 ms (in wideband) through
58 // kDelayDiffOffsetSamples. This trick does not seem to work for larger rewind
59 // values, but fortunately this is sufficient.
60 //
61 // Chromium/Linux(ChromeOS): The values we get on this platform don't correspond
62 // well to reality. The variance doesn't match the AEC's buffer changes, and the
63 // bulk values tend to be too low. However, the range across different hardware
64 // appears to be too large to choose a single value.
65 //
66 // GTP/Linux(ChromeOS): TBD, but for the moment we will trust the values.
67 #if defined(WEBRTC_CHROMIUM_BUILD) && defined(WEBRTC_MAC)
68 #define WEBRTC_UNTRUSTED_DELAY
69 #endif
70 
71 #if defined(WEBRTC_UNTRUSTED_DELAY) && defined(WEBRTC_MAC)
72 static const int kDelayDiffOffsetSamples = -160;
73 #else
74 // Not enabled for now.
75 static const int kDelayDiffOffsetSamples = 0;
76 #endif
77 
78 #if defined(WEBRTC_MAC)
79 static const int kFixedDelayMs = 20;
80 #else
81 static const int kFixedDelayMs = 50;
82 #endif
83 #if !defined(WEBRTC_UNTRUSTED_DELAY)
84 static const int kMinTrustedDelayMs = 20;
85 #endif
86 static const int kMaxTrustedDelayMs = 500;
87 
88 // Maximum length of resampled signal. Must be an integer multiple of frames
89 // (ceil(1/(1 + MIN_SKEW)*2) + 1)*FRAME_LEN
90 // The factor of 2 handles wb, and the + 1 is as a safety margin
91 // TODO(bjornv): Replace with kResamplerBufferSize
92 #define MAX_RESAMP_LEN (5 * FRAME_LEN)
93 
94 static const int kMaxBufSizeStart = 62;  // In partitions
95 static const int sampMsNb = 8;           // samples per ms in nb
96 static const int initCheck = 42;
97 
98 #ifdef WEBRTC_AEC_DEBUG_DUMP
99 int webrtc_aec_instance_count = 0;
100 #endif
101 
102 // Estimates delay to set the position of the far-end buffer read pointer
103 // (controlled by knownDelay)
104 static void EstBufDelayNormal(Aec* aecInst);
105 static void EstBufDelayExtended(Aec* aecInst);
106 static int ProcessNormal(Aec* self,
107                          const float* const* near,
108                          size_t num_bands,
109                          float* const* out,
110                          size_t num_samples,
111                          int16_t reported_delay_ms,
112                          int32_t skew);
113 static void ProcessExtended(Aec* self,
114                             const float* const* near,
115                             size_t num_bands,
116                             float* const* out,
117                             size_t num_samples,
118                             int16_t reported_delay_ms,
119                             int32_t skew);
120 
WebRtcAec_Create()121 void* WebRtcAec_Create() {
122   Aec* aecpc = malloc(sizeof(Aec));
123 
124   if (!aecpc) {
125     return NULL;
126   }
127 
128   aecpc->aec = WebRtcAec_CreateAec();
129   if (!aecpc->aec) {
130     WebRtcAec_Free(aecpc);
131     return NULL;
132   }
133   aecpc->resampler = WebRtcAec_CreateResampler();
134   if (!aecpc->resampler) {
135     WebRtcAec_Free(aecpc);
136     return NULL;
137   }
138   // Create far-end pre-buffer. The buffer size has to be large enough for
139   // largest possible drift compensation (kResamplerBufferSize) + "almost" an
140   // FFT buffer (PART_LEN2 - 1).
141   aecpc->far_pre_buf =
142       WebRtc_CreateBuffer(PART_LEN2 + kResamplerBufferSize, sizeof(float));
143   if (!aecpc->far_pre_buf) {
144     WebRtcAec_Free(aecpc);
145     return NULL;
146   }
147 
148   aecpc->initFlag = 0;
149 
150 #ifdef WEBRTC_AEC_DEBUG_DUMP
151   {
152     char filename[64];
153     sprintf(filename, "aec_buf%d.dat", webrtc_aec_instance_count);
154     aecpc->bufFile = fopen(filename, "wb");
155     sprintf(filename, "aec_skew%d.dat", webrtc_aec_instance_count);
156     aecpc->skewFile = fopen(filename, "wb");
157     sprintf(filename, "aec_delay%d.dat", webrtc_aec_instance_count);
158     aecpc->delayFile = fopen(filename, "wb");
159     webrtc_aec_instance_count++;
160   }
161 #endif
162 
163   return aecpc;
164 }
165 
WebRtcAec_Free(void * aecInst)166 void WebRtcAec_Free(void* aecInst) {
167   Aec* aecpc = aecInst;
168 
169   if (aecpc == NULL) {
170     return;
171   }
172 
173   WebRtc_FreeBuffer(aecpc->far_pre_buf);
174 
175 #ifdef WEBRTC_AEC_DEBUG_DUMP
176   fclose(aecpc->bufFile);
177   fclose(aecpc->skewFile);
178   fclose(aecpc->delayFile);
179 #endif
180 
181   WebRtcAec_FreeAec(aecpc->aec);
182   WebRtcAec_FreeResampler(aecpc->resampler);
183   free(aecpc);
184 }
185 
WebRtcAec_Init(void * aecInst,int32_t sampFreq,int32_t scSampFreq)186 int32_t WebRtcAec_Init(void* aecInst, int32_t sampFreq, int32_t scSampFreq) {
187   Aec* aecpc = aecInst;
188   AecConfig aecConfig;
189 
190   if (sampFreq != 8000 &&
191       sampFreq != 16000 &&
192       sampFreq != 32000 &&
193       sampFreq != 48000) {
194     return AEC_BAD_PARAMETER_ERROR;
195   }
196   aecpc->sampFreq = sampFreq;
197 
198   if (scSampFreq < 1 || scSampFreq > 96000) {
199     return AEC_BAD_PARAMETER_ERROR;
200   }
201   aecpc->scSampFreq = scSampFreq;
202 
203   // Initialize echo canceller core
204   if (WebRtcAec_InitAec(aecpc->aec, aecpc->sampFreq) == -1) {
205     return AEC_UNSPECIFIED_ERROR;
206   }
207 
208   if (WebRtcAec_InitResampler(aecpc->resampler, aecpc->scSampFreq) == -1) {
209     return AEC_UNSPECIFIED_ERROR;
210   }
211 
212   WebRtc_InitBuffer(aecpc->far_pre_buf);
213   WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN);  // Start overlap.
214 
215   aecpc->initFlag = initCheck;  // indicates that initialization has been done
216 
217   if (aecpc->sampFreq == 32000 || aecpc->sampFreq == 48000) {
218     aecpc->splitSampFreq = 16000;
219   } else {
220     aecpc->splitSampFreq = sampFreq;
221   }
222 
223   aecpc->delayCtr = 0;
224   aecpc->sampFactor = (aecpc->scSampFreq * 1.0f) / aecpc->splitSampFreq;
225   // Sampling frequency multiplier (SWB is processed as 160 frame size).
226   aecpc->rate_factor = aecpc->splitSampFreq / 8000;
227 
228   aecpc->sum = 0;
229   aecpc->counter = 0;
230   aecpc->checkBuffSize = 1;
231   aecpc->firstVal = 0;
232 
233   // We skip the startup_phase completely (setting to 0) if DA-AEC is enabled,
234   // but not extended_filter mode.
235   aecpc->startup_phase = WebRtcAec_extended_filter_enabled(aecpc->aec) ||
236       !WebRtcAec_delay_agnostic_enabled(aecpc->aec);
237   aecpc->bufSizeStart = 0;
238   aecpc->checkBufSizeCtr = 0;
239   aecpc->msInSndCardBuf = 0;
240   aecpc->filtDelay = -1;  // -1 indicates an initialized state.
241   aecpc->timeForDelayChange = 0;
242   aecpc->knownDelay = 0;
243   aecpc->lastDelayDiff = 0;
244 
245   aecpc->skewFrCtr = 0;
246   aecpc->resample = kAecFalse;
247   aecpc->highSkewCtr = 0;
248   aecpc->skew = 0;
249 
250   aecpc->farend_started = 0;
251 
252   // Default settings.
253   aecConfig.nlpMode = kAecNlpModerate;
254   aecConfig.skewMode = kAecFalse;
255   aecConfig.metricsMode = kAecFalse;
256   aecConfig.delay_logging = kAecFalse;
257 
258   if (WebRtcAec_set_config(aecpc, aecConfig) == -1) {
259     return AEC_UNSPECIFIED_ERROR;
260   }
261 
262   return 0;
263 }
264 
265 // Returns any error that is caused when buffering the
266 // far-end signal.
WebRtcAec_GetBufferFarendError(void * aecInst,const float * farend,size_t nrOfSamples)267 int32_t WebRtcAec_GetBufferFarendError(void* aecInst,
268                                        const float* farend,
269                                        size_t nrOfSamples) {
270   Aec* aecpc = aecInst;
271 
272   if (!farend)
273     return AEC_NULL_POINTER_ERROR;
274 
275   if (aecpc->initFlag != initCheck)
276     return AEC_UNINITIALIZED_ERROR;
277 
278   // number of samples == 160 for SWB input
279   if (nrOfSamples != 80 && nrOfSamples != 160)
280     return AEC_BAD_PARAMETER_ERROR;
281 
282   return 0;
283 }
284 
285 // only buffer L band for farend
WebRtcAec_BufferFarend(void * aecInst,const float * farend,size_t nrOfSamples)286 int32_t WebRtcAec_BufferFarend(void* aecInst,
287                                const float* farend,
288                                size_t nrOfSamples) {
289   Aec* aecpc = aecInst;
290   size_t newNrOfSamples = nrOfSamples;
291   float new_farend[MAX_RESAMP_LEN];
292   const float* farend_ptr = farend;
293 
294   // Get any error caused by buffering the farend signal.
295   int32_t error_code = WebRtcAec_GetBufferFarendError(aecInst, farend,
296                                                       nrOfSamples);
297 
298   if (error_code != 0)
299     return error_code;
300 
301 
302   if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) {
303     // Resample and get a new number of samples
304     WebRtcAec_ResampleLinear(aecpc->resampler,
305                              farend,
306                              nrOfSamples,
307                              aecpc->skew,
308                              new_farend,
309                              &newNrOfSamples);
310     farend_ptr = new_farend;
311   }
312 
313   aecpc->farend_started = 1;
314   WebRtcAec_SetSystemDelay(
315       aecpc->aec, WebRtcAec_system_delay(aecpc->aec) + (int)newNrOfSamples);
316 
317   // Write the time-domain data to |far_pre_buf|.
318   WebRtc_WriteBuffer(aecpc->far_pre_buf, farend_ptr, newNrOfSamples);
319 
320   // TODO(minyue): reduce to |PART_LEN| samples for each buffering, when
321   // WebRtcAec_BufferFarendPartition() is changed to take |PART_LEN| samples.
322   while (WebRtc_available_read(aecpc->far_pre_buf) >= PART_LEN2) {
323     // We have enough data to pass to the FFT, hence read PART_LEN2 samples.
324     {
325       float* ptmp = NULL;
326       float tmp[PART_LEN2];
327       WebRtc_ReadBuffer(aecpc->far_pre_buf, (void**)&ptmp, tmp, PART_LEN2);
328       WebRtcAec_BufferFarendPartition(aecpc->aec, ptmp);
329     }
330 
331     // Rewind |far_pre_buf| PART_LEN samples for overlap before continuing.
332     WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN);
333   }
334 
335   return 0;
336 }
337 
WebRtcAec_Process(void * aecInst,const float * const * nearend,size_t num_bands,float * const * out,size_t nrOfSamples,int16_t msInSndCardBuf,int32_t skew)338 int32_t WebRtcAec_Process(void* aecInst,
339                           const float* const* nearend,
340                           size_t num_bands,
341                           float* const* out,
342                           size_t nrOfSamples,
343                           int16_t msInSndCardBuf,
344                           int32_t skew) {
345   Aec* aecpc = aecInst;
346   int32_t retVal = 0;
347 
348   if (out == NULL) {
349     return AEC_NULL_POINTER_ERROR;
350   }
351 
352   if (aecpc->initFlag != initCheck) {
353     return AEC_UNINITIALIZED_ERROR;
354   }
355 
356   // number of samples == 160 for SWB input
357   if (nrOfSamples != 80 && nrOfSamples != 160) {
358     return AEC_BAD_PARAMETER_ERROR;
359   }
360 
361   if (msInSndCardBuf < 0) {
362     msInSndCardBuf = 0;
363     retVal = AEC_BAD_PARAMETER_WARNING;
364   } else if (msInSndCardBuf > kMaxTrustedDelayMs) {
365     // The clamping is now done in ProcessExtended/Normal().
366     retVal = AEC_BAD_PARAMETER_WARNING;
367   }
368 
369   // This returns the value of aec->extended_filter_enabled.
370   if (WebRtcAec_extended_filter_enabled(aecpc->aec)) {
371     ProcessExtended(aecpc,
372                     nearend,
373                     num_bands,
374                     out,
375                     nrOfSamples,
376                     msInSndCardBuf,
377                     skew);
378   } else {
379     retVal = ProcessNormal(aecpc,
380                            nearend,
381                            num_bands,
382                            out,
383                            nrOfSamples,
384                            msInSndCardBuf,
385                            skew);
386   }
387 
388 #ifdef WEBRTC_AEC_DEBUG_DUMP
389   {
390     int16_t far_buf_size_ms = (int16_t)(WebRtcAec_system_delay(aecpc->aec) /
391                                         (sampMsNb * aecpc->rate_factor));
392     (void)fwrite(&far_buf_size_ms, 2, 1, aecpc->bufFile);
393     (void)fwrite(
394         &aecpc->knownDelay, sizeof(aecpc->knownDelay), 1, aecpc->delayFile);
395   }
396 #endif
397 
398   return retVal;
399 }
400 
WebRtcAec_set_config(void * handle,AecConfig config)401 int WebRtcAec_set_config(void* handle, AecConfig config) {
402   Aec* self = (Aec*)handle;
403   if (self->initFlag != initCheck) {
404     return AEC_UNINITIALIZED_ERROR;
405   }
406 
407   if (config.skewMode != kAecFalse && config.skewMode != kAecTrue) {
408     return AEC_BAD_PARAMETER_ERROR;
409   }
410   self->skewMode = config.skewMode;
411 
412   if (config.nlpMode != kAecNlpConservative &&
413       config.nlpMode != kAecNlpModerate &&
414       config.nlpMode != kAecNlpAggressive) {
415     return AEC_BAD_PARAMETER_ERROR;
416   }
417 
418   if (config.metricsMode != kAecFalse && config.metricsMode != kAecTrue) {
419     return AEC_BAD_PARAMETER_ERROR;
420   }
421 
422   if (config.delay_logging != kAecFalse && config.delay_logging != kAecTrue) {
423     return AEC_BAD_PARAMETER_ERROR;
424   }
425 
426   WebRtcAec_SetConfigCore(
427       self->aec, config.nlpMode, config.metricsMode, config.delay_logging);
428   return 0;
429 }
430 
WebRtcAec_get_echo_status(void * handle,int * status)431 int WebRtcAec_get_echo_status(void* handle, int* status) {
432   Aec* self = (Aec*)handle;
433   if (status == NULL) {
434     return AEC_NULL_POINTER_ERROR;
435   }
436   if (self->initFlag != initCheck) {
437     return AEC_UNINITIALIZED_ERROR;
438   }
439 
440   *status = WebRtcAec_echo_state(self->aec);
441 
442   return 0;
443 }
444 
WebRtcAec_GetMetrics(void * handle,AecMetrics * metrics)445 int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics) {
446   const float kUpWeight = 0.7f;
447   float dtmp;
448   int stmp;
449   Aec* self = (Aec*)handle;
450   Stats erl;
451   Stats erle;
452   Stats a_nlp;
453 
454   if (handle == NULL) {
455     return -1;
456   }
457   if (metrics == NULL) {
458     return AEC_NULL_POINTER_ERROR;
459   }
460   if (self->initFlag != initCheck) {
461     return AEC_UNINITIALIZED_ERROR;
462   }
463 
464   WebRtcAec_GetEchoStats(self->aec, &erl, &erle, &a_nlp);
465 
466   // ERL
467   metrics->erl.instant = (int)erl.instant;
468 
469   if ((erl.himean > kOffsetLevel) && (erl.average > kOffsetLevel)) {
470     // Use a mix between regular average and upper part average.
471     dtmp = kUpWeight * erl.himean + (1 - kUpWeight) * erl.average;
472     metrics->erl.average = (int)dtmp;
473   } else {
474     metrics->erl.average = kOffsetLevel;
475   }
476 
477   metrics->erl.max = (int)erl.max;
478 
479   if (erl.min < (kOffsetLevel * (-1))) {
480     metrics->erl.min = (int)erl.min;
481   } else {
482     metrics->erl.min = kOffsetLevel;
483   }
484 
485   // ERLE
486   metrics->erle.instant = (int)erle.instant;
487 
488   if ((erle.himean > kOffsetLevel) && (erle.average > kOffsetLevel)) {
489     // Use a mix between regular average and upper part average.
490     dtmp = kUpWeight * erle.himean + (1 - kUpWeight) * erle.average;
491     metrics->erle.average = (int)dtmp;
492   } else {
493     metrics->erle.average = kOffsetLevel;
494   }
495 
496   metrics->erle.max = (int)erle.max;
497 
498   if (erle.min < (kOffsetLevel * (-1))) {
499     metrics->erle.min = (int)erle.min;
500   } else {
501     metrics->erle.min = kOffsetLevel;
502   }
503 
504   // RERL
505   if ((metrics->erl.average > kOffsetLevel) &&
506       (metrics->erle.average > kOffsetLevel)) {
507     stmp = metrics->erl.average + metrics->erle.average;
508   } else {
509     stmp = kOffsetLevel;
510   }
511   metrics->rerl.average = stmp;
512 
513   // No other statistics needed, but returned for completeness.
514   metrics->rerl.instant = stmp;
515   metrics->rerl.max = stmp;
516   metrics->rerl.min = stmp;
517 
518   // A_NLP
519   metrics->aNlp.instant = (int)a_nlp.instant;
520 
521   if ((a_nlp.himean > kOffsetLevel) && (a_nlp.average > kOffsetLevel)) {
522     // Use a mix between regular average and upper part average.
523     dtmp = kUpWeight * a_nlp.himean + (1 - kUpWeight) * a_nlp.average;
524     metrics->aNlp.average = (int)dtmp;
525   } else {
526     metrics->aNlp.average = kOffsetLevel;
527   }
528 
529   metrics->aNlp.max = (int)a_nlp.max;
530 
531   if (a_nlp.min < (kOffsetLevel * (-1))) {
532     metrics->aNlp.min = (int)a_nlp.min;
533   } else {
534     metrics->aNlp.min = kOffsetLevel;
535   }
536 
537   return 0;
538 }
539 
WebRtcAec_GetDelayMetrics(void * handle,int * median,int * std,float * fraction_poor_delays)540 int WebRtcAec_GetDelayMetrics(void* handle,
541                               int* median,
542                               int* std,
543                               float* fraction_poor_delays) {
544   Aec* self = handle;
545   if (median == NULL) {
546     return AEC_NULL_POINTER_ERROR;
547   }
548   if (std == NULL) {
549     return AEC_NULL_POINTER_ERROR;
550   }
551   if (self->initFlag != initCheck) {
552     return AEC_UNINITIALIZED_ERROR;
553   }
554   if (WebRtcAec_GetDelayMetricsCore(self->aec, median, std,
555                                     fraction_poor_delays) ==
556       -1) {
557     // Logging disabled.
558     return AEC_UNSUPPORTED_FUNCTION_ERROR;
559   }
560 
561   return 0;
562 }
563 
564 
WebRtcAec_aec_core(void * handle)565 AecCore* WebRtcAec_aec_core(void* handle) {
566   if (!handle) {
567     return NULL;
568   }
569   return ((Aec*)handle)->aec;
570 }
571 
ProcessNormal(Aec * aecpc,const float * const * nearend,size_t num_bands,float * const * out,size_t nrOfSamples,int16_t msInSndCardBuf,int32_t skew)572 static int ProcessNormal(Aec* aecpc,
573                          const float* const* nearend,
574                          size_t num_bands,
575                          float* const* out,
576                          size_t nrOfSamples,
577                          int16_t msInSndCardBuf,
578                          int32_t skew) {
579   int retVal = 0;
580   size_t i;
581   size_t nBlocks10ms;
582   // Limit resampling to doubling/halving of signal
583   const float minSkewEst = -0.5f;
584   const float maxSkewEst = 1.0f;
585 
586   msInSndCardBuf =
587       msInSndCardBuf > kMaxTrustedDelayMs ? kMaxTrustedDelayMs : msInSndCardBuf;
588   // TODO(andrew): we need to investigate if this +10 is really wanted.
589   msInSndCardBuf += 10;
590   aecpc->msInSndCardBuf = msInSndCardBuf;
591 
592   if (aecpc->skewMode == kAecTrue) {
593     if (aecpc->skewFrCtr < 25) {
594       aecpc->skewFrCtr++;
595     } else {
596       retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew);
597       if (retVal == -1) {
598         aecpc->skew = 0;
599         retVal = AEC_BAD_PARAMETER_WARNING;
600       }
601 
602       aecpc->skew /= aecpc->sampFactor * nrOfSamples;
603 
604       if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) {
605         aecpc->resample = kAecFalse;
606       } else {
607         aecpc->resample = kAecTrue;
608       }
609 
610       if (aecpc->skew < minSkewEst) {
611         aecpc->skew = minSkewEst;
612       } else if (aecpc->skew > maxSkewEst) {
613         aecpc->skew = maxSkewEst;
614       }
615 
616 #ifdef WEBRTC_AEC_DEBUG_DUMP
617       (void)fwrite(&aecpc->skew, sizeof(aecpc->skew), 1, aecpc->skewFile);
618 #endif
619     }
620   }
621 
622   nBlocks10ms = nrOfSamples / (FRAME_LEN * aecpc->rate_factor);
623 
624   if (aecpc->startup_phase) {
625     for (i = 0; i < num_bands; ++i) {
626       // Only needed if they don't already point to the same place.
627       if (nearend[i] != out[i]) {
628         memcpy(out[i], nearend[i], sizeof(nearend[i][0]) * nrOfSamples);
629       }
630     }
631 
632     // The AEC is in the start up mode
633     // AEC is disabled until the system delay is OK
634 
635     // Mechanism to ensure that the system delay is reasonably stable.
636     if (aecpc->checkBuffSize) {
637       aecpc->checkBufSizeCtr++;
638       // Before we fill up the far-end buffer we require the system delay
639       // to be stable (+/-8 ms) compared to the first value. This
640       // comparison is made during the following 6 consecutive 10 ms
641       // blocks. If it seems to be stable then we start to fill up the
642       // far-end buffer.
643       if (aecpc->counter == 0) {
644         aecpc->firstVal = aecpc->msInSndCardBuf;
645         aecpc->sum = 0;
646       }
647 
648       if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) <
649           WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) {
650         aecpc->sum += aecpc->msInSndCardBuf;
651         aecpc->counter++;
652       } else {
653         aecpc->counter = 0;
654       }
655 
656       if (aecpc->counter * nBlocks10ms >= 6) {
657         // The far-end buffer size is determined in partitions of
658         // PART_LEN samples. Use 75% of the average value of the system
659         // delay as buffer size to start with.
660         aecpc->bufSizeStart =
661             WEBRTC_SPL_MIN((3 * aecpc->sum * aecpc->rate_factor * 8) /
662                                (4 * aecpc->counter * PART_LEN),
663                            kMaxBufSizeStart);
664         // Buffer size has now been determined.
665         aecpc->checkBuffSize = 0;
666       }
667 
668       if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) {
669         // For really bad systems, don't disable the echo canceller for
670         // more than 0.5 sec.
671         aecpc->bufSizeStart = WEBRTC_SPL_MIN(
672             (aecpc->msInSndCardBuf * aecpc->rate_factor * 3) / 40,
673             kMaxBufSizeStart);
674         aecpc->checkBuffSize = 0;
675       }
676     }
677 
678     // If |checkBuffSize| changed in the if-statement above.
679     if (!aecpc->checkBuffSize) {
680       // The system delay is now reasonably stable (or has been unstable
681       // for too long). When the far-end buffer is filled with
682       // approximately the same amount of data as reported by the system
683       // we end the startup phase.
684       int overhead_elements =
685           WebRtcAec_system_delay(aecpc->aec) / PART_LEN - aecpc->bufSizeStart;
686       if (overhead_elements == 0) {
687         // Enable the AEC
688         aecpc->startup_phase = 0;
689       } else if (overhead_elements > 0) {
690         // TODO(bjornv): Do we need a check on how much we actually
691         // moved the read pointer? It should always be possible to move
692         // the pointer |overhead_elements| since we have only added data
693         // to the buffer and no delay compensation nor AEC processing
694         // has been done.
695         WebRtcAec_MoveFarReadPtr(aecpc->aec, overhead_elements);
696 
697         // Enable the AEC
698         aecpc->startup_phase = 0;
699       }
700     }
701   } else {
702     // AEC is enabled.
703     EstBufDelayNormal(aecpc);
704 
705     // Call the AEC.
706     // TODO(bjornv): Re-structure such that we don't have to pass
707     // |aecpc->knownDelay| as input. Change name to something like
708     // |system_buffer_diff|.
709     WebRtcAec_ProcessFrames(aecpc->aec,
710                             nearend,
711                             num_bands,
712                             nrOfSamples,
713                             aecpc->knownDelay,
714                             out);
715   }
716 
717   return retVal;
718 }
719 
ProcessExtended(Aec * self,const float * const * near,size_t num_bands,float * const * out,size_t num_samples,int16_t reported_delay_ms,int32_t skew)720 static void ProcessExtended(Aec* self,
721                             const float* const* near,
722                             size_t num_bands,
723                             float* const* out,
724                             size_t num_samples,
725                             int16_t reported_delay_ms,
726                             int32_t skew) {
727   size_t i;
728   const int delay_diff_offset = kDelayDiffOffsetSamples;
729 #if defined(WEBRTC_UNTRUSTED_DELAY)
730   reported_delay_ms = kFixedDelayMs;
731 #else
732   // This is the usual mode where we trust the reported system delay values.
733   // Due to the longer filter, we no longer add 10 ms to the reported delay
734   // to reduce chance of non-causality. Instead we apply a minimum here to avoid
735   // issues with the read pointer jumping around needlessly.
736   reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs
737                           ? kMinTrustedDelayMs
738                           : reported_delay_ms;
739   // If the reported delay appears to be bogus, we attempt to recover by using
740   // the measured fixed delay values. We use >= here because higher layers
741   // may already clamp to this maximum value, and we would otherwise not
742   // detect it here.
743   reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs
744                           ? kFixedDelayMs
745                           : reported_delay_ms;
746 #endif
747   self->msInSndCardBuf = reported_delay_ms;
748 
749   if (!self->farend_started) {
750     for (i = 0; i < num_bands; ++i) {
751       // Only needed if they don't already point to the same place.
752       if (near[i] != out[i]) {
753         memcpy(out[i], near[i], sizeof(near[i][0]) * num_samples);
754       }
755     }
756     return;
757   }
758   if (self->startup_phase) {
759     // In the extended mode, there isn't a startup "phase", just a special
760     // action on the first frame. In the trusted delay case, we'll take the
761     // current reported delay, unless it's less then our conservative
762     // measurement.
763     int startup_size_ms =
764         reported_delay_ms < kFixedDelayMs ? kFixedDelayMs : reported_delay_ms;
765 #if defined(WEBRTC_ANDROID)
766     int target_delay = startup_size_ms * self->rate_factor * 8;
767 #else
768     // To avoid putting the AEC in a non-causal state we're being slightly
769     // conservative and scale by 2. On Android we use a fixed delay and
770     // therefore there is no need to scale the target_delay.
771     int target_delay = startup_size_ms * self->rate_factor * 8 / 2;
772 #endif
773     int overhead_elements =
774         (WebRtcAec_system_delay(self->aec) - target_delay) / PART_LEN;
775     WebRtcAec_MoveFarReadPtr(self->aec, overhead_elements);
776     self->startup_phase = 0;
777   }
778 
779   EstBufDelayExtended(self);
780 
781   {
782     // |delay_diff_offset| gives us the option to manually rewind the delay on
783     // very low delay platforms which can't be expressed purely through
784     // |reported_delay_ms|.
785     const int adjusted_known_delay =
786         WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset);
787 
788     WebRtcAec_ProcessFrames(self->aec,
789                             near,
790                             num_bands,
791                             num_samples,
792                             adjusted_known_delay,
793                             out);
794   }
795 }
796 
EstBufDelayNormal(Aec * aecpc)797 static void EstBufDelayNormal(Aec* aecpc) {
798   int nSampSndCard = aecpc->msInSndCardBuf * sampMsNb * aecpc->rate_factor;
799   int current_delay = nSampSndCard - WebRtcAec_system_delay(aecpc->aec);
800   int delay_difference = 0;
801 
802   // Before we proceed with the delay estimate filtering we:
803   // 1) Compensate for the frame that will be read.
804   // 2) Compensate for drift resampling.
805   // 3) Compensate for non-causality if needed, since the estimated delay can't
806   //    be negative.
807 
808   // 1) Compensating for the frame(s) that will be read/processed.
809   current_delay += FRAME_LEN * aecpc->rate_factor;
810 
811   // 2) Account for resampling frame delay.
812   if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) {
813     current_delay -= kResamplingDelay;
814   }
815 
816   // 3) Compensate for non-causality, if needed, by flushing one block.
817   if (current_delay < PART_LEN) {
818     current_delay += WebRtcAec_MoveFarReadPtr(aecpc->aec, 1) * PART_LEN;
819   }
820 
821   // We use -1 to signal an initialized state in the "extended" implementation;
822   // compensate for that.
823   aecpc->filtDelay = aecpc->filtDelay < 0 ? 0 : aecpc->filtDelay;
824   aecpc->filtDelay =
825       WEBRTC_SPL_MAX(0, (short)(0.8 * aecpc->filtDelay + 0.2 * current_delay));
826 
827   delay_difference = aecpc->filtDelay - aecpc->knownDelay;
828   if (delay_difference > 224) {
829     if (aecpc->lastDelayDiff < 96) {
830       aecpc->timeForDelayChange = 0;
831     } else {
832       aecpc->timeForDelayChange++;
833     }
834   } else if (delay_difference < 96 && aecpc->knownDelay > 0) {
835     if (aecpc->lastDelayDiff > 224) {
836       aecpc->timeForDelayChange = 0;
837     } else {
838       aecpc->timeForDelayChange++;
839     }
840   } else {
841     aecpc->timeForDelayChange = 0;
842   }
843   aecpc->lastDelayDiff = delay_difference;
844 
845   if (aecpc->timeForDelayChange > 25) {
846     aecpc->knownDelay = WEBRTC_SPL_MAX((int)aecpc->filtDelay - 160, 0);
847   }
848 }
849 
EstBufDelayExtended(Aec * self)850 static void EstBufDelayExtended(Aec* self) {
851   int reported_delay = self->msInSndCardBuf * sampMsNb * self->rate_factor;
852   int current_delay = reported_delay - WebRtcAec_system_delay(self->aec);
853   int delay_difference = 0;
854 
855   // Before we proceed with the delay estimate filtering we:
856   // 1) Compensate for the frame that will be read.
857   // 2) Compensate for drift resampling.
858   // 3) Compensate for non-causality if needed, since the estimated delay can't
859   //    be negative.
860 
861   // 1) Compensating for the frame(s) that will be read/processed.
862   current_delay += FRAME_LEN * self->rate_factor;
863 
864   // 2) Account for resampling frame delay.
865   if (self->skewMode == kAecTrue && self->resample == kAecTrue) {
866     current_delay -= kResamplingDelay;
867   }
868 
869   // 3) Compensate for non-causality, if needed, by flushing two blocks.
870   if (current_delay < PART_LEN) {
871     current_delay += WebRtcAec_MoveFarReadPtr(self->aec, 2) * PART_LEN;
872   }
873 
874   if (self->filtDelay == -1) {
875     self->filtDelay = WEBRTC_SPL_MAX(0, 0.5 * current_delay);
876   } else {
877     self->filtDelay = WEBRTC_SPL_MAX(
878         0, (short)(0.95 * self->filtDelay + 0.05 * current_delay));
879   }
880 
881   delay_difference = self->filtDelay - self->knownDelay;
882   if (delay_difference > 384) {
883     if (self->lastDelayDiff < 128) {
884       self->timeForDelayChange = 0;
885     } else {
886       self->timeForDelayChange++;
887     }
888   } else if (delay_difference < 128 && self->knownDelay > 0) {
889     if (self->lastDelayDiff > 384) {
890       self->timeForDelayChange = 0;
891     } else {
892       self->timeForDelayChange++;
893     }
894   } else {
895     self->timeForDelayChange = 0;
896   }
897   self->lastDelayDiff = delay_difference;
898 
899   if (self->timeForDelayChange > 25) {
900     self->knownDelay = WEBRTC_SPL_MAX((int)self->filtDelay - 256, 0);
901   }
902 }
903