1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12 #define WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
13 
14 #include <list>
15 #include <string>
16 #include <vector>
17 
18 #include "webrtc/base/criticalsection.h"
19 #include "webrtc/base/scoped_ptr.h"
20 #include "webrtc/base/thread_annotations.h"
21 #include "webrtc/modules/audio_processing/audio_buffer.h"
22 #include "webrtc/modules/audio_processing/include/audio_processing.h"
23 #include "webrtc/system_wrappers/include/file_wrapper.h"
24 
25 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
26 // Files generated at build-time by the protobuf compiler.
27 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
28 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
29 #else
30 #include "webrtc/audio_processing/debug.pb.h"
31 #endif
32 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
33 
34 namespace webrtc {
35 
36 class AgcManagerDirect;
37 class AudioConverter;
38 
39 template<typename T>
40 class Beamformer;
41 
42 class AudioProcessingImpl : public AudioProcessing {
43  public:
44   // Methods forcing APM to run in a single-threaded manner.
45   // Acquires both the render and capture locks.
46   explicit AudioProcessingImpl(const Config& config);
47   // AudioProcessingImpl takes ownership of beamformer.
48   AudioProcessingImpl(const Config& config, Beamformer<float>* beamformer);
49   virtual ~AudioProcessingImpl();
50   int Initialize() override;
51   int Initialize(int input_sample_rate_hz,
52                  int output_sample_rate_hz,
53                  int reverse_sample_rate_hz,
54                  ChannelLayout input_layout,
55                  ChannelLayout output_layout,
56                  ChannelLayout reverse_layout) override;
57   int Initialize(const ProcessingConfig& processing_config) override;
58   void SetExtraOptions(const Config& config) override;
59   void UpdateHistogramsOnCallEnd() override;
60   int StartDebugRecording(const char filename[kMaxFilenameSize]) override;
61   int StartDebugRecording(FILE* handle) override;
62   int StartDebugRecordingForPlatformFile(rtc::PlatformFile handle) override;
63   int StopDebugRecording() override;
64 
65   // Capture-side exclusive methods possibly running APM in a
66   // multi-threaded manner. Acquire the capture lock.
67   int ProcessStream(AudioFrame* frame) override;
68   int ProcessStream(const float* const* src,
69                     size_t samples_per_channel,
70                     int input_sample_rate_hz,
71                     ChannelLayout input_layout,
72                     int output_sample_rate_hz,
73                     ChannelLayout output_layout,
74                     float* const* dest) override;
75   int ProcessStream(const float* const* src,
76                     const StreamConfig& input_config,
77                     const StreamConfig& output_config,
78                     float* const* dest) override;
79   void set_output_will_be_muted(bool muted) override;
80   int set_stream_delay_ms(int delay) override;
81   void set_delay_offset_ms(int offset) override;
82   int delay_offset_ms() const override;
83   void set_stream_key_pressed(bool key_pressed) override;
84   int input_sample_rate_hz() const override;
85 
86   // Render-side exclusive methods possibly running APM in a
87   // multi-threaded manner. Acquire the render lock.
88   int AnalyzeReverseStream(AudioFrame* frame) override;
89   int ProcessReverseStream(AudioFrame* frame) override;
90   int AnalyzeReverseStream(const float* const* data,
91                            size_t samples_per_channel,
92                            int sample_rate_hz,
93                            ChannelLayout layout) override;
94   int ProcessReverseStream(const float* const* src,
95                            const StreamConfig& reverse_input_config,
96                            const StreamConfig& reverse_output_config,
97                            float* const* dest) override;
98 
99   // Methods only accessed from APM submodules or
100   // from AudioProcessing tests in a single-threaded manner.
101   // Hence there is no need for locks in these.
102   int proc_sample_rate_hz() const override;
103   int proc_split_sample_rate_hz() const override;
104   size_t num_input_channels() const override;
105   size_t num_proc_channels() const override;
106   size_t num_output_channels() const override;
107   size_t num_reverse_channels() const override;
108   int stream_delay_ms() const override;
109   bool was_stream_delay_set() const override
110       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
111 
112   // Methods returning pointers to APM submodules.
113   // No locks are aquired in those, as those locks
114   // would offer no protection (the submodules are
115   // created only once in a single-treaded manner
116   // during APM creation).
117   EchoCancellation* echo_cancellation() const override;
118   EchoControlMobile* echo_control_mobile() const override;
119   GainControl* gain_control() const override;
120   HighPassFilter* high_pass_filter() const override;
121   LevelEstimator* level_estimator() const override;
122   NoiseSuppression* noise_suppression() const override;
123   VoiceDetection* voice_detection() const override;
124 
125  protected:
126   // Overridden in a mock.
127   virtual int InitializeLocked()
128       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
129 
130  private:
131   struct ApmPublicSubmodules;
132   struct ApmPrivateSubmodules;
133 
134 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
135   // State for the debug dump.
136   struct ApmDebugDumpThreadState {
ApmDebugDumpThreadStateApmDebugDumpThreadState137     ApmDebugDumpThreadState() : event_msg(new audioproc::Event()) {}
138     rtc::scoped_ptr<audioproc::Event> event_msg;  // Protobuf message.
139     std::string event_str;  // Memory for protobuf serialization.
140 
141     // Serialized string of last saved APM configuration.
142     std::string last_serialized_config;
143   };
144 
145   struct ApmDebugDumpState {
ApmDebugDumpStateApmDebugDumpState146     ApmDebugDumpState() : debug_file(FileWrapper::Create()) {}
147     rtc::scoped_ptr<FileWrapper> debug_file;
148     ApmDebugDumpThreadState render;
149     ApmDebugDumpThreadState capture;
150   };
151 #endif
152 
153   // Method for modifying the formats struct that are called from both
154   // the render and capture threads. The check for whether modifications
155   // are needed is done while holding the render lock only, thereby avoiding
156   // that the capture thread blocks the render thread.
157   // The struct is modified in a single-threaded manner by holding both the
158   // render and capture locks.
159   int MaybeInitialize(const ProcessingConfig& config)
160       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
161 
162   int MaybeInitializeRender(const ProcessingConfig& processing_config)
163       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
164 
165   int MaybeInitializeCapture(const ProcessingConfig& processing_config)
166       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
167 
168   // Method for checking for the need of conversion. Accesses the formats
169   // structs in a read manner but the requirement for the render lock to be held
170   // was added as it currently anyway is always called in that manner.
171   bool rev_conversion_needed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
172   bool render_check_rev_conversion_needed() const
173       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
174 
175   // Methods requiring APM running in a single-threaded manner.
176   // Are called with both the render and capture locks already
177   // acquired.
178   void InitializeExperimentalAgc()
179       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
180   void InitializeTransient()
181       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
182   void InitializeBeamformer()
183       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
184   void InitializeIntelligibility()
185       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
186   void InitializeHighPassFilter()
187       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
188   void InitializeNoiseSuppression()
189       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
190   void InitializeLevelEstimator()
191       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
192   void InitializeVoiceDetection()
193       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
194   int InitializeLocked(const ProcessingConfig& config)
195       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
196 
197   // Capture-side exclusive methods possibly running APM in a multi-threaded
198   // manner that are called with the render lock already acquired.
199   int ProcessStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
200   bool output_copy_needed(bool is_data_processed) const
201       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
202   bool is_data_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
203   bool synthesis_needed(bool is_data_processed) const
204       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
205   bool analysis_needed(bool is_data_processed) const
206       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
207   void MaybeUpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
208 
209   // Render-side exclusive methods possibly running APM in a multi-threaded
210   // manner that are called with the render lock already acquired.
211   // TODO(ekm): Remove once all clients updated to new interface.
212   int AnalyzeReverseStreamLocked(const float* const* src,
213                                  const StreamConfig& input_config,
214                                  const StreamConfig& output_config)
215       EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
216   bool is_rev_processed() const EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
217   int ProcessReverseStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
218 
219 // Debug dump methods that are internal and called without locks.
220 // TODO(peah): Make thread safe.
221 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
222   // TODO(andrew): make this more graceful. Ideally we would split this stuff
223   // out into a separate class with an "enabled" and "disabled" implementation.
224   static int WriteMessageToDebugFile(FileWrapper* debug_file,
225                                      rtc::CriticalSection* crit_debug,
226                                      ApmDebugDumpThreadState* debug_state);
227   int WriteInitMessage() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
228 
229   // Writes Config message. If not |forced|, only writes the current config if
230   // it is different from the last saved one; if |forced|, writes the config
231   // regardless of the last saved.
232   int WriteConfigMessage(bool forced) EXCLUSIVE_LOCKS_REQUIRED(crit_capture_)
233       EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
234 
235   // Critical section.
236   mutable rtc::CriticalSection crit_debug_;
237 
238   // Debug dump state.
239   ApmDebugDumpState debug_dump_;
240 #endif
241 
242   // Critical sections.
243   mutable rtc::CriticalSection crit_render_ ACQUIRED_BEFORE(crit_capture_);
244   mutable rtc::CriticalSection crit_capture_;
245 
246   // Structs containing the pointers to the submodules.
247   rtc::scoped_ptr<ApmPublicSubmodules> public_submodules_;
248   rtc::scoped_ptr<ApmPrivateSubmodules> private_submodules_
249       GUARDED_BY(crit_capture_);
250 
251   // State that is written to while holding both the render and capture locks
252   // but can be read without any lock being held.
253   // As this is only accessed internally of APM, and all internal methods in APM
254   // either are holding the render or capture locks, this construct is safe as
255   // it is not possible to read the variables while writing them.
256   struct ApmFormatState {
ApmFormatStateApmFormatState257     ApmFormatState()
258         :  // Format of processing streams at input/output call sites.
259           api_format({{{kSampleRate16kHz, 1, false},
260                        {kSampleRate16kHz, 1, false},
261                        {kSampleRate16kHz, 1, false},
262                        {kSampleRate16kHz, 1, false}}}),
263           rev_proc_format(kSampleRate16kHz, 1) {}
264     ProcessingConfig api_format;
265     StreamConfig rev_proc_format;
266   } formats_;
267 
268   // APM constants.
269   const struct ApmConstants {
ApmConstantsApmConstants270     ApmConstants(int agc_startup_min_volume,
271                  bool use_new_agc,
272                  bool intelligibility_enabled)
273         :  // Format of processing streams at input/output call sites.
274           agc_startup_min_volume(agc_startup_min_volume),
275           use_new_agc(use_new_agc),
276           intelligibility_enabled(intelligibility_enabled) {}
277     int agc_startup_min_volume;
278     bool use_new_agc;
279     bool intelligibility_enabled;
280   } constants_;
281 
282   struct ApmCaptureState {
ApmCaptureStateApmCaptureState283     ApmCaptureState(bool transient_suppressor_enabled,
284                     const std::vector<Point>& array_geometry,
285                     SphericalPointf target_direction)
286         : aec_system_delay_jumps(-1),
287           delay_offset_ms(0),
288           was_stream_delay_set(false),
289           last_stream_delay_ms(0),
290           last_aec_system_delay_ms(0),
291           stream_delay_jumps(-1),
292           output_will_be_muted(false),
293           key_pressed(false),
294           transient_suppressor_enabled(transient_suppressor_enabled),
295           array_geometry(array_geometry),
296           target_direction(target_direction),
297           fwd_proc_format(kSampleRate16kHz),
298           split_rate(kSampleRate16kHz) {}
299     int aec_system_delay_jumps;
300     int delay_offset_ms;
301     bool was_stream_delay_set;
302     int last_stream_delay_ms;
303     int last_aec_system_delay_ms;
304     int stream_delay_jumps;
305     bool output_will_be_muted;
306     bool key_pressed;
307     bool transient_suppressor_enabled;
308     std::vector<Point> array_geometry;
309     SphericalPointf target_direction;
310     rtc::scoped_ptr<AudioBuffer> capture_audio;
311     // Only the rate and samples fields of fwd_proc_format_ are used because the
312     // forward processing number of channels is mutable and is tracked by the
313     // capture_audio_.
314     StreamConfig fwd_proc_format;
315     int split_rate;
316   } capture_ GUARDED_BY(crit_capture_);
317 
318   struct ApmCaptureNonLockedState {
ApmCaptureNonLockedStateApmCaptureNonLockedState319     ApmCaptureNonLockedState(bool beamformer_enabled)
320         : fwd_proc_format(kSampleRate16kHz),
321           split_rate(kSampleRate16kHz),
322           stream_delay_ms(0),
323           beamformer_enabled(beamformer_enabled) {}
324     // Only the rate and samples fields of fwd_proc_format_ are used because the
325     // forward processing number of channels is mutable and is tracked by the
326     // capture_audio_.
327     StreamConfig fwd_proc_format;
328     int split_rate;
329     int stream_delay_ms;
330     bool beamformer_enabled;
331   } capture_nonlocked_;
332 
333   struct ApmRenderState {
334     rtc::scoped_ptr<AudioConverter> render_converter;
335     rtc::scoped_ptr<AudioBuffer> render_audio;
336   } render_ GUARDED_BY(crit_render_);
337 };
338 
339 }  // namespace webrtc
340 
341 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
342