1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 12 #define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 13 14 #include <stddef.h> 15 16 #include <memory> 17 #include <vector> 18 19 #include "api/array_view.h" 20 #include "api/audio/echo_canceller3_config.h" 21 #include "api/audio/echo_control.h" 22 #include "modules/audio_processing/aec3/api_call_jitter_metrics.h" 23 #include "modules/audio_processing/aec3/block_delay_buffer.h" 24 #include "modules/audio_processing/aec3/block_framer.h" 25 #include "modules/audio_processing/aec3/block_processor.h" 26 #include "modules/audio_processing/aec3/frame_blocker.h" 27 #include "modules/audio_processing/audio_buffer.h" 28 #include "modules/audio_processing/logging/apm_data_dumper.h" 29 #include "rtc_base/checks.h" 30 #include "rtc_base/race_checker.h" 31 #include "rtc_base/swap_queue.h" 32 #include "rtc_base/thread_annotations.h" 33 34 namespace webrtc { 35 36 // Method for adjusting config parameter dependencies. 37 // Only to be used externally to AEC3 for testing purposes. 38 // TODO(webrtc:5298): Move this to a separate file. 39 EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config); 40 41 // Functor for verifying the invariance of the frames being put into the render 42 // queue. 43 class Aec3RenderQueueItemVerifier { 44 public: Aec3RenderQueueItemVerifier(size_t num_bands,size_t num_channels,size_t frame_length)45 Aec3RenderQueueItemVerifier(size_t num_bands, 46 size_t num_channels, 47 size_t frame_length) 48 : num_bands_(num_bands), 49 num_channels_(num_channels), 50 frame_length_(frame_length) {} 51 operator()52 bool operator()(const std::vector<std::vector<std::vector<float>>>& v) const { 53 if (v.size() != num_bands_) { 54 return false; 55 } 56 for (const auto& band : v) { 57 if (band.size() != num_channels_) { 58 return false; 59 } 60 for (const auto& channel : band) { 61 if (channel.size() != frame_length_) { 62 return false; 63 } 64 } 65 } 66 return true; 67 } 68 69 private: 70 const size_t num_bands_; 71 const size_t num_channels_; 72 const size_t frame_length_; 73 }; 74 75 // Main class for the echo canceller3. 76 // It does 4 things: 77 // -Receives 10 ms frames of band-split audio. 78 // -Provides the lower level echo canceller functionality with 79 // blocks of 64 samples of audio data. 80 // -Partially handles the jitter in the render and capture API 81 // call sequence. 82 // 83 // The class is supposed to be used in a non-concurrent manner apart from the 84 // AnalyzeRender call which can be called concurrently with the other methods. 85 class EchoCanceller3 : public EchoControl { 86 public: 87 // Normal c-tor to use. 88 EchoCanceller3(const EchoCanceller3Config& config, 89 int sample_rate_hz, 90 size_t num_render_channels, 91 size_t num_capture_channels); 92 // Testing c-tor that is used only for testing purposes. 93 EchoCanceller3(const EchoCanceller3Config& config, 94 int sample_rate_hz, 95 size_t num_render_channels, 96 size_t num_capture_channels, 97 std::unique_ptr<BlockProcessor> block_processor); 98 ~EchoCanceller3() override; 99 EchoCanceller3(const EchoCanceller3&) = delete; 100 EchoCanceller3& operator=(const EchoCanceller3&) = delete; 101 102 // Analyzes and stores an internal copy of the split-band domain render 103 // signal. AnalyzeRender(AudioBuffer * render)104 void AnalyzeRender(AudioBuffer* render) override { AnalyzeRender(*render); } 105 // Analyzes the full-band domain capture signal to detect signal saturation. AnalyzeCapture(AudioBuffer * capture)106 void AnalyzeCapture(AudioBuffer* capture) override { 107 AnalyzeCapture(*capture); 108 } 109 // Processes the split-band domain capture signal in order to remove any echo 110 // present in the signal. 111 void ProcessCapture(AudioBuffer* capture, bool level_change) override; 112 // As above, but also returns the linear filter output. 113 void ProcessCapture(AudioBuffer* capture, 114 AudioBuffer* linear_output, 115 bool level_change) override; 116 // Collect current metrics from the echo canceller. 117 Metrics GetMetrics() const override; 118 // Provides an optional external estimate of the audio buffer delay. 119 void SetAudioBufferDelay(int delay_ms) override; 120 121 bool ActiveProcessing() const override; 122 123 // Signals whether an external detector has detected echo leakage from the 124 // echo canceller. 125 // Note that in the case echo leakage has been flagged, it should be unflagged 126 // once it is no longer occurring. UpdateEchoLeakageStatus(bool leakage_detected)127 void UpdateEchoLeakageStatus(bool leakage_detected) { 128 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); 129 block_processor_->UpdateEchoLeakageStatus(leakage_detected); 130 } 131 132 // Produces a default configuration that is suitable for a certain combination 133 // of render and capture channels. 134 static EchoCanceller3Config CreateDefaultConfig(size_t num_render_channels, 135 size_t num_capture_channels); 136 137 private: 138 class RenderWriter; 139 140 // Empties the render SwapQueue. 141 void EmptyRenderQueue(); 142 143 // Analyzes and stores an internal copy of the split-band domain render 144 // signal. 145 void AnalyzeRender(const AudioBuffer& render); 146 // Analyzes the full-band domain capture signal to detect signal saturation. 147 void AnalyzeCapture(const AudioBuffer& capture); 148 149 rtc::RaceChecker capture_race_checker_; 150 rtc::RaceChecker render_race_checker_; 151 152 // State that is accessed by the AnalyzeRender call. 153 std::unique_ptr<RenderWriter> render_writer_ 154 RTC_GUARDED_BY(render_race_checker_); 155 156 // State that may be accessed by the capture thread. 157 static int instance_count_; 158 std::unique_ptr<ApmDataDumper> data_dumper_; 159 const EchoCanceller3Config config_; 160 const int sample_rate_hz_; 161 const int num_bands_; 162 const size_t num_render_channels_; 163 const size_t num_capture_channels_; 164 std::unique_ptr<BlockFramer> linear_output_framer_ 165 RTC_GUARDED_BY(capture_race_checker_); 166 BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); 167 FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); 168 FrameBlocker render_blocker_ RTC_GUARDED_BY(capture_race_checker_); 169 SwapQueue<std::vector<std::vector<std::vector<float>>>, 170 Aec3RenderQueueItemVerifier> 171 render_transfer_queue_; 172 std::unique_ptr<BlockProcessor> block_processor_ 173 RTC_GUARDED_BY(capture_race_checker_); 174 std::vector<std::vector<std::vector<float>>> render_queue_output_frame_ 175 RTC_GUARDED_BY(capture_race_checker_); 176 bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = 177 false; 178 std::vector<std::vector<std::vector<float>>> render_block_ 179 RTC_GUARDED_BY(capture_race_checker_); 180 std::unique_ptr<std::vector<std::vector<std::vector<float>>>> 181 linear_output_block_ RTC_GUARDED_BY(capture_race_checker_); 182 std::vector<std::vector<std::vector<float>>> capture_block_ 183 RTC_GUARDED_BY(capture_race_checker_); 184 std::vector<std::vector<rtc::ArrayView<float>>> render_sub_frame_view_ 185 RTC_GUARDED_BY(capture_race_checker_); 186 std::vector<std::vector<rtc::ArrayView<float>>> linear_output_sub_frame_view_ 187 RTC_GUARDED_BY(capture_race_checker_); 188 std::vector<std::vector<rtc::ArrayView<float>>> capture_sub_frame_view_ 189 RTC_GUARDED_BY(capture_race_checker_); 190 std::unique_ptr<BlockDelayBuffer> block_delay_buffer_ 191 RTC_GUARDED_BY(capture_race_checker_); 192 ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_); 193 }; 194 } // namespace webrtc 195 196 #endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_ 197