1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
12 #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
13 
14 #include <string>
15 
16 #include "webrtc/base/constructormagic.h"
17 #include "webrtc/base/scoped_ptr.h"
18 #include "webrtc/base/thread_annotations.h"
19 #include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h"
20 #include "webrtc/modules/audio_coding/neteq/defines.h"
21 #include "webrtc/modules/audio_coding/neteq/include/neteq.h"
22 #include "webrtc/modules/audio_coding/neteq/packet.h"  // Declare PacketList.
23 #include "webrtc/modules/audio_coding/neteq/random_vector.h"
24 #include "webrtc/modules/audio_coding/neteq/rtcp.h"
25 #include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
26 #include "webrtc/typedefs.h"
27 
28 namespace webrtc {
29 
30 // Forward declarations.
31 class Accelerate;
32 class BackgroundNoise;
33 class BufferLevelFilter;
34 class ComfortNoise;
35 class CriticalSectionWrapper;
36 class DecisionLogic;
37 class DecoderDatabase;
38 class DelayManager;
39 class DelayPeakDetector;
40 class DtmfBuffer;
41 class DtmfToneGenerator;
42 class Expand;
43 class Merge;
44 class Nack;
45 class Normal;
46 class PacketBuffer;
47 class PayloadSplitter;
48 class PostDecodeVad;
49 class PreemptiveExpand;
50 class RandomVector;
51 class SyncBuffer;
52 class TimestampScaler;
53 struct AccelerateFactory;
54 struct DtmfEvent;
55 struct ExpandFactory;
56 struct PreemptiveExpandFactory;
57 
58 class NetEqImpl : public webrtc::NetEq {
59  public:
60   // Creates a new NetEqImpl object. The object will assume ownership of all
61   // injected dependencies, and will delete them when done.
62   NetEqImpl(const NetEq::Config& config,
63             BufferLevelFilter* buffer_level_filter,
64             DecoderDatabase* decoder_database,
65             DelayManager* delay_manager,
66             DelayPeakDetector* delay_peak_detector,
67             DtmfBuffer* dtmf_buffer,
68             DtmfToneGenerator* dtmf_tone_generator,
69             PacketBuffer* packet_buffer,
70             PayloadSplitter* payload_splitter,
71             TimestampScaler* timestamp_scaler,
72             AccelerateFactory* accelerate_factory,
73             ExpandFactory* expand_factory,
74             PreemptiveExpandFactory* preemptive_expand_factory,
75             bool create_components = true);
76 
77   ~NetEqImpl() override;
78 
79   // Inserts a new packet into NetEq. The |receive_timestamp| is an indication
80   // of the time when the packet was received, and should be measured with
81   // the same tick rate as the RTP timestamp of the current payload.
82   // Returns 0 on success, -1 on failure.
83   int InsertPacket(const WebRtcRTPHeader& rtp_header,
84                    rtc::ArrayView<const uint8_t> payload,
85                    uint32_t receive_timestamp) override;
86 
87   // Inserts a sync-packet into packet queue. Sync-packets are decoded to
88   // silence and are intended to keep AV-sync intact in an event of long packet
89   // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq
90   // might insert sync-packet when they observe that buffer level of NetEq is
91   // decreasing below a certain threshold, defined by the application.
92   // Sync-packets should have the same payload type as the last audio payload
93   // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change
94   // can be implied by inserting a sync-packet.
95   // Returns kOk on success, kFail on failure.
96   int InsertSyncPacket(const WebRtcRTPHeader& rtp_header,
97                        uint32_t receive_timestamp) override;
98 
99   // Instructs NetEq to deliver 10 ms of audio data. The data is written to
100   // |output_audio|, which can hold (at least) |max_length| elements.
101   // The number of channels that were written to the output is provided in
102   // the output variable |num_channels|, and each channel contains
103   // |samples_per_channel| elements. If more than one channel is written,
104   // the samples are interleaved.
105   // The speech type is written to |type|, if |type| is not NULL.
106   // Returns kOK on success, or kFail in case of an error.
107   int GetAudio(size_t max_length,
108                int16_t* output_audio,
109                size_t* samples_per_channel,
110                size_t* num_channels,
111                NetEqOutputType* type) override;
112 
113   int RegisterPayloadType(NetEqDecoder codec,
114                           const std::string& codec_name,
115                           uint8_t rtp_payload_type) override;
116 
117   int RegisterExternalDecoder(AudioDecoder* decoder,
118                               NetEqDecoder codec,
119                               const std::string& codec_name,
120                               uint8_t rtp_payload_type,
121                               int sample_rate_hz) override;
122 
123   // Removes |rtp_payload_type| from the codec database. Returns 0 on success,
124   // -1 on failure.
125   int RemovePayloadType(uint8_t rtp_payload_type) override;
126 
127   bool SetMinimumDelay(int delay_ms) override;
128 
129   bool SetMaximumDelay(int delay_ms) override;
130 
131   int LeastRequiredDelayMs() const override;
132 
133   int SetTargetDelay() override;
134 
135   int TargetDelay() override;
136 
137   int CurrentDelayMs() const override;
138 
139   // Sets the playout mode to |mode|.
140   // Deprecated.
141   // TODO(henrik.lundin) Delete.
142   void SetPlayoutMode(NetEqPlayoutMode mode) override;
143 
144   // Returns the current playout mode.
145   // Deprecated.
146   // TODO(henrik.lundin) Delete.
147   NetEqPlayoutMode PlayoutMode() const override;
148 
149   // Writes the current network statistics to |stats|. The statistics are reset
150   // after the call.
151   int NetworkStatistics(NetEqNetworkStatistics* stats) override;
152 
153   // Writes the current RTCP statistics to |stats|. The statistics are reset
154   // and a new report period is started with the call.
155   void GetRtcpStatistics(RtcpStatistics* stats) override;
156 
157   // Same as RtcpStatistics(), but does not reset anything.
158   void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override;
159 
160   // Enables post-decode VAD. When enabled, GetAudio() will return
161   // kOutputVADPassive when the signal contains no speech.
162   void EnableVad() override;
163 
164   // Disables post-decode VAD.
165   void DisableVad() override;
166 
167   bool GetPlayoutTimestamp(uint32_t* timestamp) override;
168 
169   int last_output_sample_rate_hz() const override;
170 
171   int SetTargetNumberOfChannels() override;
172 
173   int SetTargetSampleRate() override;
174 
175   // Returns the error code for the last occurred error. If no error has
176   // occurred, 0 is returned.
177   int LastError() const override;
178 
179   // Returns the error code last returned by a decoder (audio or comfort noise).
180   // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check
181   // this method to get the decoder's error code.
182   int LastDecoderError() override;
183 
184   // Flushes both the packet buffer and the sync buffer.
185   void FlushBuffers() override;
186 
187   void PacketBufferStatistics(int* current_num_packets,
188                               int* max_num_packets) const override;
189 
190   void EnableNack(size_t max_nack_list_size) override;
191 
192   void DisableNack() override;
193 
194   std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
195 
196   // This accessor method is only intended for testing purposes.
197   const SyncBuffer* sync_buffer_for_test() const;
198 
199  protected:
200   static const int kOutputSizeMs = 10;
201   static const size_t kMaxFrameSize = 2880;  // 60 ms @ 48 kHz.
202   // TODO(hlundin): Provide a better value for kSyncBufferSize.
203   static const size_t kSyncBufferSize = 2 * kMaxFrameSize;
204 
205   // Inserts a new packet into NetEq. This is used by the InsertPacket method
206   // above. Returns 0 on success, otherwise an error code.
207   // TODO(hlundin): Merge this with InsertPacket above?
208   int InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
209                            rtc::ArrayView<const uint8_t> payload,
210                            uint32_t receive_timestamp,
211                            bool is_sync_packet)
212       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
213 
214   // Delivers 10 ms of audio data. The data is written to |output|, which can
215   // hold (at least) |max_length| elements. The number of channels that were
216   // written to the output is provided in the output variable |num_channels|,
217   // and each channel contains |samples_per_channel| elements. If more than one
218   // channel is written, the samples are interleaved.
219   // Returns 0 on success, otherwise an error code.
220   int GetAudioInternal(size_t max_length,
221                        int16_t* output,
222                        size_t* samples_per_channel,
223                        size_t* num_channels)
224       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
225 
226   // Provides a decision to the GetAudioInternal method. The decision what to
227   // do is written to |operation|. Packets to decode are written to
228   // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When
229   // DTMF should be played, |play_dtmf| is set to true by the method.
230   // Returns 0 on success, otherwise an error code.
231   int GetDecision(Operations* operation,
232                   PacketList* packet_list,
233                   DtmfEvent* dtmf_event,
234                   bool* play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
235 
236   // Decodes the speech packets in |packet_list|, and writes the results to
237   // |decoded_buffer|, which is allocated to hold |decoded_buffer_length|
238   // elements. The length of the decoded data is written to |decoded_length|.
239   // The speech type -- speech or (codec-internal) comfort noise -- is written
240   // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389
241   // comfort noise, those are not decoded.
242   int Decode(PacketList* packet_list,
243              Operations* operation,
244              int* decoded_length,
245              AudioDecoder::SpeechType* speech_type)
246       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
247 
248   // Sub-method to Decode(). Performs codec internal CNG.
249   int DecodeCng(AudioDecoder* decoder, int* decoded_length,
250                 AudioDecoder::SpeechType* speech_type)
251       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
252 
253   // Sub-method to Decode(). Performs the actual decoding.
254   int DecodeLoop(PacketList* packet_list,
255                  const Operations& operation,
256                  AudioDecoder* decoder,
257                  int* decoded_length,
258                  AudioDecoder::SpeechType* speech_type)
259       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
260 
261   // Sub-method which calls the Normal class to perform the normal operation.
262   void DoNormal(const int16_t* decoded_buffer,
263                 size_t decoded_length,
264                 AudioDecoder::SpeechType speech_type,
265                 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
266 
267   // Sub-method which calls the Merge class to perform the merge operation.
268   void DoMerge(int16_t* decoded_buffer,
269                size_t decoded_length,
270                AudioDecoder::SpeechType speech_type,
271                bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
272 
273   // Sub-method which calls the Expand class to perform the expand operation.
274   int DoExpand(bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
275 
276   // Sub-method which calls the Accelerate class to perform the accelerate
277   // operation.
278   int DoAccelerate(int16_t* decoded_buffer,
279                    size_t decoded_length,
280                    AudioDecoder::SpeechType speech_type,
281                    bool play_dtmf,
282                    bool fast_accelerate) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
283 
284   // Sub-method which calls the PreemptiveExpand class to perform the
285   // preemtive expand operation.
286   int DoPreemptiveExpand(int16_t* decoded_buffer,
287                          size_t decoded_length,
288                          AudioDecoder::SpeechType speech_type,
289                          bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
290 
291   // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort
292   // noise. |packet_list| can either contain one SID frame to update the
293   // noise parameters, or no payload at all, in which case the previously
294   // received parameters are used.
295   int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf)
296       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
297 
298   // Calls the audio decoder to generate codec-internal comfort noise when
299   // no packet was received.
300   void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length)
301       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
302 
303   // Calls the DtmfToneGenerator class to generate DTMF tones.
304   int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf)
305       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
306 
307   // Produces packet-loss concealment using alternative methods. If the codec
308   // has an internal PLC, it is called to generate samples. Otherwise, the
309   // method performs zero-stuffing.
310   void DoAlternativePlc(bool increase_timestamp)
311       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
312 
313   // Overdub DTMF on top of |output|.
314   int DtmfOverdub(const DtmfEvent& dtmf_event,
315                   size_t num_channels,
316                   int16_t* output) const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
317 
318   // Extracts packets from |packet_buffer_| to produce at least
319   // |required_samples| samples. The packets are inserted into |packet_list|.
320   // Returns the number of samples that the packets in the list will produce, or
321   // -1 in case of an error.
322   int ExtractPackets(size_t required_samples, PacketList* packet_list)
323       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
324 
325   // Resets various variables and objects to new values based on the sample rate
326   // |fs_hz| and |channels| number audio channels.
327   void SetSampleRateAndChannels(int fs_hz, size_t channels)
328       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
329 
330   // Returns the output type for the audio produced by the latest call to
331   // GetAudio().
332   NetEqOutputType LastOutputType() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
333 
334   // Updates Expand and Merge.
335   virtual void UpdatePlcComponents(int fs_hz, size_t channels)
336       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
337 
338   // Creates DecisionLogic object with the mode given by |playout_mode_|.
339   virtual void CreateDecisionLogic() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
340 
341   const rtc::scoped_ptr<CriticalSectionWrapper> crit_sect_;
342   const rtc::scoped_ptr<BufferLevelFilter> buffer_level_filter_
343       GUARDED_BY(crit_sect_);
344   const rtc::scoped_ptr<DecoderDatabase> decoder_database_
345       GUARDED_BY(crit_sect_);
346   const rtc::scoped_ptr<DelayManager> delay_manager_ GUARDED_BY(crit_sect_);
347   const rtc::scoped_ptr<DelayPeakDetector> delay_peak_detector_
348       GUARDED_BY(crit_sect_);
349   const rtc::scoped_ptr<DtmfBuffer> dtmf_buffer_ GUARDED_BY(crit_sect_);
350   const rtc::scoped_ptr<DtmfToneGenerator> dtmf_tone_generator_
351       GUARDED_BY(crit_sect_);
352   const rtc::scoped_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_);
353   const rtc::scoped_ptr<PayloadSplitter> payload_splitter_
354       GUARDED_BY(crit_sect_);
355   const rtc::scoped_ptr<TimestampScaler> timestamp_scaler_
356       GUARDED_BY(crit_sect_);
357   const rtc::scoped_ptr<PostDecodeVad> vad_ GUARDED_BY(crit_sect_);
358   const rtc::scoped_ptr<ExpandFactory> expand_factory_ GUARDED_BY(crit_sect_);
359   const rtc::scoped_ptr<AccelerateFactory> accelerate_factory_
360       GUARDED_BY(crit_sect_);
361   const rtc::scoped_ptr<PreemptiveExpandFactory> preemptive_expand_factory_
362       GUARDED_BY(crit_sect_);
363 
364   rtc::scoped_ptr<BackgroundNoise> background_noise_ GUARDED_BY(crit_sect_);
365   rtc::scoped_ptr<DecisionLogic> decision_logic_ GUARDED_BY(crit_sect_);
366   rtc::scoped_ptr<AudioMultiVector> algorithm_buffer_ GUARDED_BY(crit_sect_);
367   rtc::scoped_ptr<SyncBuffer> sync_buffer_ GUARDED_BY(crit_sect_);
368   rtc::scoped_ptr<Expand> expand_ GUARDED_BY(crit_sect_);
369   rtc::scoped_ptr<Normal> normal_ GUARDED_BY(crit_sect_);
370   rtc::scoped_ptr<Merge> merge_ GUARDED_BY(crit_sect_);
371   rtc::scoped_ptr<Accelerate> accelerate_ GUARDED_BY(crit_sect_);
372   rtc::scoped_ptr<PreemptiveExpand> preemptive_expand_ GUARDED_BY(crit_sect_);
373   RandomVector random_vector_ GUARDED_BY(crit_sect_);
374   rtc::scoped_ptr<ComfortNoise> comfort_noise_ GUARDED_BY(crit_sect_);
375   Rtcp rtcp_ GUARDED_BY(crit_sect_);
376   StatisticsCalculator stats_ GUARDED_BY(crit_sect_);
377   int fs_hz_ GUARDED_BY(crit_sect_);
378   int fs_mult_ GUARDED_BY(crit_sect_);
379   int last_output_sample_rate_hz_ GUARDED_BY(crit_sect_);
380   size_t output_size_samples_ GUARDED_BY(crit_sect_);
381   size_t decoder_frame_length_ GUARDED_BY(crit_sect_);
382   Modes last_mode_ GUARDED_BY(crit_sect_);
383   rtc::scoped_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_);
384   size_t decoded_buffer_length_ GUARDED_BY(crit_sect_);
385   rtc::scoped_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_);
386   uint32_t playout_timestamp_ GUARDED_BY(crit_sect_);
387   bool new_codec_ GUARDED_BY(crit_sect_);
388   uint32_t timestamp_ GUARDED_BY(crit_sect_);
389   bool reset_decoder_ GUARDED_BY(crit_sect_);
390   uint8_t current_rtp_payload_type_ GUARDED_BY(crit_sect_);
391   uint8_t current_cng_rtp_payload_type_ GUARDED_BY(crit_sect_);
392   uint32_t ssrc_ GUARDED_BY(crit_sect_);
393   bool first_packet_ GUARDED_BY(crit_sect_);
394   int error_code_ GUARDED_BY(crit_sect_);  // Store last error code.
395   int decoder_error_code_ GUARDED_BY(crit_sect_);
396   const BackgroundNoiseMode background_noise_mode_ GUARDED_BY(crit_sect_);
397   NetEqPlayoutMode playout_mode_ GUARDED_BY(crit_sect_);
398   bool enable_fast_accelerate_ GUARDED_BY(crit_sect_);
399   rtc::scoped_ptr<Nack> nack_ GUARDED_BY(crit_sect_);
400   bool nack_enabled_ GUARDED_BY(crit_sect_);
401 
402  private:
403   RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl);
404 };
405 
406 }  // namespace webrtc
407 #endif  // WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_
408