• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  /*
2   *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3   *
4   *  Use of this source code is governed by a BSD-style license
5   *  that can be found in the LICENSE file in the root of the source
6   *  tree. An additional intellectual property rights grant can be found
7   *  in the file PATENTS.  All contributing project authors may
8   *  be found in the AUTHORS file in the root of the source tree.
9   */
10  
11  #ifndef WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
12  #define WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
13  
14  #include <list>
15  #include <map>
16  #include <set>
17  #include <vector>
18  
19  #include "webrtc/base/constructormagic.h"
20  #include "webrtc/base/thread_annotations.h"
21  #include "webrtc/modules/include/module_common_types.h"
22  #include "webrtc/modules/video_coding/include/video_coding.h"
23  #include "webrtc/modules/video_coding/include/video_coding_defines.h"
24  #include "webrtc/modules/video_coding/decoding_state.h"
25  #include "webrtc/modules/video_coding/inter_frame_delay.h"
26  #include "webrtc/modules/video_coding/jitter_buffer_common.h"
27  #include "webrtc/modules/video_coding/jitter_estimator.h"
28  #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
29  #include "webrtc/typedefs.h"
30  
31  namespace webrtc {
32  
33  enum VCMNackMode { kNack, kNoNack };
34  
35  // forward declarations
36  class Clock;
37  class EventFactory;
38  class EventWrapper;
39  class VCMFrameBuffer;
40  class VCMPacket;
41  class VCMEncodedFrame;
42  
43  typedef std::list<VCMFrameBuffer*> UnorderedFrameList;
44  
45  struct VCMJitterSample {
VCMJitterSampleVCMJitterSample46    VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {}
47    uint32_t timestamp;
48    uint32_t frame_size;
49    int64_t latest_packet_time;
50  };
51  
52  class TimestampLessThan {
53   public:
operator()54    bool operator()(uint32_t timestamp1, uint32_t timestamp2) const {
55      return IsNewerTimestamp(timestamp2, timestamp1);
56    }
57  };
58  
59  class FrameList
60      : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> {
61   public:
62    void InsertFrame(VCMFrameBuffer* frame);
63    VCMFrameBuffer* PopFrame(uint32_t timestamp);
64    VCMFrameBuffer* Front() const;
65    VCMFrameBuffer* Back() const;
66    int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
67                                   UnorderedFrameList* free_frames);
68    void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
69                                 UnorderedFrameList* free_frames);
70    void Reset(UnorderedFrameList* free_frames);
71  };
72  
73  class Vp9SsMap {
74   public:
75    typedef std::map<uint32_t, GofInfoVP9, TimestampLessThan> SsMap;
76    bool Insert(const VCMPacket& packet);
77    void Reset();
78  
79    // Removes SS data that are older than |timestamp|.
80    // The |timestamp| should be an old timestamp, i.e. packets with older
81    // timestamps should no longer be inserted.
82    void RemoveOld(uint32_t timestamp);
83  
84    bool UpdatePacket(VCMPacket* packet);
85    void UpdateFrames(FrameList* frames);
86  
87    // Public for testing.
88    // Returns an iterator to the corresponding SS data for the input |timestamp|.
89    bool Find(uint32_t timestamp, SsMap::iterator* it);
90  
91   private:
92    // These two functions are called by RemoveOld.
93    // Checks if it is time to do a clean up (done each kSsCleanupIntervalSec).
94    bool TimeForCleanup(uint32_t timestamp) const;
95  
96    // Advances the oldest SS data to handle timestamp wrap in cases where SS data
97    // are received very seldom (e.g. only once in beginning, second when
98    // IsNewerTimestamp is not true).
99    void AdvanceFront(uint32_t timestamp);
100  
101    SsMap ss_map_;
102  };
103  
104  class VCMJitterBuffer {
105   public:
106    VCMJitterBuffer(Clock* clock, rtc::scoped_ptr<EventWrapper> event);
107  
108    ~VCMJitterBuffer();
109  
110    // Initializes and starts jitter buffer.
111    void Start();
112  
113    // Signals all internal events and stops the jitter buffer.
114    void Stop();
115  
116    // Returns true if the jitter buffer is running.
117    bool Running() const;
118  
119    // Empty the jitter buffer of all its data.
120    void Flush();
121  
122    // Get the number of received frames, by type, since the jitter buffer
123    // was started.
124    FrameCounts FrameStatistics() const;
125  
126    // The number of packets discarded by the jitter buffer because the decoder
127    // won't be able to decode them.
128    int num_not_decodable_packets() const;
129  
130    // Gets number of packets received.
131    int num_packets() const;
132  
133    // Gets number of duplicated packets received.
134    int num_duplicated_packets() const;
135  
136    // Gets number of packets discarded by the jitter buffer.
137    int num_discarded_packets() const;
138  
139    // Statistics, Calculate frame and bit rates.
140    void IncomingRateStatistics(unsigned int* framerate, unsigned int* bitrate);
141  
142    // Checks if the packet sequence will be complete if the next frame would be
143    // grabbed for decoding. That is, if a frame has been lost between the
144    // last decoded frame and the next, or if the next frame is missing one
145    // or more packets.
146    bool CompleteSequenceWithNextFrame();
147  
148    // Wait |max_wait_time_ms| for a complete frame to arrive.
149    // The function returns true once such a frame is found, its corresponding
150    // timestamp is returned. Otherwise, returns false.
151    bool NextCompleteTimestamp(uint32_t max_wait_time_ms, uint32_t* timestamp);
152  
153    // Locates a frame for decoding (even an incomplete) without delay.
154    // The function returns true once such a frame is found, its corresponding
155    // timestamp is returned. Otherwise, returns false.
156    bool NextMaybeIncompleteTimestamp(uint32_t* timestamp);
157  
158    // Extract frame corresponding to input timestamp.
159    // Frame will be set to a decoding state.
160    VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp);
161  
162    // Releases a frame returned from the jitter buffer, should be called when
163    // done with decoding.
164    void ReleaseFrame(VCMEncodedFrame* frame);
165  
166    // Returns the time in ms when the latest packet was inserted into the frame.
167    // Retransmitted is set to true if any of the packets belonging to the frame
168    // has been retransmitted.
169    int64_t LastPacketTime(const VCMEncodedFrame* frame,
170                           bool* retransmitted) const;
171  
172    // Inserts a packet into a frame returned from GetFrame().
173    // If the return value is <= 0, |frame| is invalidated and the pointer must
174    // be dropped after this function returns.
175    VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted);
176  
177    // Returns the estimated jitter in milliseconds.
178    uint32_t EstimatedJitterMs();
179  
180    // Updates the round-trip time estimate.
181    void UpdateRtt(int64_t rtt_ms);
182  
183    // Set the NACK mode. |high_rtt_nack_threshold_ms| is an RTT threshold in ms
184    // above which NACK will be disabled if the NACK mode is |kNack|, -1 meaning
185    // that NACK is always enabled in the |kNack| mode.
186    // |low_rtt_nack_threshold_ms| is an RTT threshold in ms below which we expect
187    // to rely on NACK only, and therefore are using larger buffers to have time
188    // to wait for retransmissions.
189    void SetNackMode(VCMNackMode mode,
190                     int64_t low_rtt_nack_threshold_ms,
191                     int64_t high_rtt_nack_threshold_ms);
192  
193    void SetNackSettings(size_t max_nack_list_size,
194                         int max_packet_age_to_nack,
195                         int max_incomplete_time_ms);
196  
197    // Returns the current NACK mode.
198    VCMNackMode nack_mode() const;
199  
200    // Returns a list of the sequence numbers currently missing.
201    std::vector<uint16_t> GetNackList(bool* request_key_frame);
202  
203    // Set decode error mode - Should not be changed in the middle of the
204    // session. Changes will not influence frames already in the buffer.
205    void SetDecodeErrorMode(VCMDecodeErrorMode error_mode);
206    int64_t LastDecodedTimestamp() const;
decode_error_mode()207    VCMDecodeErrorMode decode_error_mode() const { return decode_error_mode_; }
208  
209    // Used to compute time of complete continuous frames. Returns the timestamps
210    // corresponding to the start and end of the continuous complete buffer.
211    void RenderBufferSize(uint32_t* timestamp_start, uint32_t* timestamp_end);
212  
213    void RegisterStatsCallback(VCMReceiveStatisticsCallback* callback);
214  
215   private:
216    class SequenceNumberLessThan {
217     public:
operator()218      bool operator()(const uint16_t& sequence_number1,
219                      const uint16_t& sequence_number2) const {
220        return IsNewerSequenceNumber(sequence_number2, sequence_number1);
221      }
222    };
223    typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet;
224  
225    // Gets the frame assigned to the timestamp of the packet. May recycle
226    // existing frames if no free frames are available. Returns an error code if
227    // failing, or kNoError on success. |frame_list| contains which list the
228    // packet was in, or NULL if it was not in a FrameList (a new frame).
229    VCMFrameBufferEnum GetFrame(const VCMPacket& packet,
230                                VCMFrameBuffer** frame,
231                                FrameList** frame_list)
232        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
233  
234    // Returns true if |frame| is continuous in |decoding_state|, not taking
235    // decodable frames into account.
236    bool IsContinuousInState(const VCMFrameBuffer& frame,
237                             const VCMDecodingState& decoding_state) const
238        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
239    // Returns true if |frame| is continuous in the |last_decoded_state_|, taking
240    // all decodable frames into account.
241    bool IsContinuous(const VCMFrameBuffer& frame) const
242        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
243    // Looks for frames in |incomplete_frames_| which are continuous in the
244    // provided |decoded_state|. Starts the search from the timestamp of
245    // |decoded_state|.
246    void FindAndInsertContinuousFramesWithState(
247        const VCMDecodingState& decoded_state)
248        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
249    // Looks for frames in |incomplete_frames_| which are continuous in
250    // |last_decoded_state_| taking all decodable frames into account. Starts
251    // the search from |new_frame|.
252    void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame)
253        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
254    VCMFrameBuffer* NextFrame() const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
255    // Returns true if the NACK list was updated to cover sequence numbers up to
256    // |sequence_number|. If false a key frame is needed to get into a state where
257    // we can continue decoding.
258    bool UpdateNackList(uint16_t sequence_number)
259        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
260    bool TooLargeNackList() const;
261    // Returns true if the NACK list was reduced without problem. If false a key
262    // frame is needed to get into a state where we can continue decoding.
263    bool HandleTooLargeNackList() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
264    bool MissingTooOldPacket(uint16_t latest_sequence_number) const
265        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
266    // Returns true if the too old packets was successfully removed from the NACK
267    // list. If false, a key frame is needed to get into a state where we can
268    // continue decoding.
269    bool HandleTooOldPackets(uint16_t latest_sequence_number)
270        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
271    // Drops all packets in the NACK list up until |last_decoded_sequence_number|.
272    void DropPacketsFromNackList(uint16_t last_decoded_sequence_number);
273  
274    void ReleaseFrameIfNotDecoding(VCMFrameBuffer* frame);
275  
276    // Gets an empty frame, creating a new frame if necessary (i.e. increases
277    // jitter buffer size).
278    VCMFrameBuffer* GetEmptyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
279  
280    // Attempts to increase the size of the jitter buffer. Returns true on
281    // success, false otherwise.
282    bool TryToIncreaseJitterBufferSize() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
283  
284    // Recycles oldest frames until a key frame is found. Used if jitter buffer is
285    // completely full. Returns true if a key frame was found.
286    bool RecycleFramesUntilKeyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
287  
288    // Updates the frame statistics.
289    // Counts only complete frames, so decodable incomplete frames will not be
290    // counted.
291    void CountFrame(const VCMFrameBuffer& frame)
292        EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
293  
294    // Update rolling average of packets per frame.
295    void UpdateAveragePacketsPerFrame(int current_number_packets_);
296  
297    // Cleans the frame list in the JB from old/empty frames.
298    // Should only be called prior to actual use.
299    void CleanUpOldOrEmptyFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
300  
301    // Returns true if |packet| is likely to have been retransmitted.
302    bool IsPacketRetransmitted(const VCMPacket& packet) const;
303  
304    // The following three functions update the jitter estimate with the
305    // payload size, receive time and RTP timestamp of a frame.
306    void UpdateJitterEstimate(const VCMJitterSample& sample,
307                              bool incomplete_frame);
308    void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame);
309    void UpdateJitterEstimate(int64_t latest_packet_time_ms,
310                              uint32_t timestamp,
311                              unsigned int frame_size,
312                              bool incomplete_frame);
313  
314    // Returns true if we should wait for retransmissions, false otherwise.
315    bool WaitForRetransmissions();
316  
317    int NonContinuousOrIncompleteDuration() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
318  
319    uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const;
320  
321    void UpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
322  
323    Clock* clock_;
324    // If we are running (have started) or not.
325    bool running_;
326    CriticalSectionWrapper* crit_sect_;
327    // Event to signal when we have a frame ready for decoder.
328    rtc::scoped_ptr<EventWrapper> frame_event_;
329    // Number of allocated frames.
330    int max_number_of_frames_;
331    UnorderedFrameList free_frames_ GUARDED_BY(crit_sect_);
332    FrameList decodable_frames_ GUARDED_BY(crit_sect_);
333    FrameList incomplete_frames_ GUARDED_BY(crit_sect_);
334    VCMDecodingState last_decoded_state_ GUARDED_BY(crit_sect_);
335    bool first_packet_since_reset_;
336  
337    // Statistics.
338    VCMReceiveStatisticsCallback* stats_callback_ GUARDED_BY(crit_sect_);
339    // Frame counts for each type (key, delta, ...)
340    FrameCounts receive_statistics_;
341    // Latest calculated frame rates of incoming stream.
342    unsigned int incoming_frame_rate_;
343    unsigned int incoming_frame_count_;
344    int64_t time_last_incoming_frame_count_;
345    unsigned int incoming_bit_count_;
346    unsigned int incoming_bit_rate_;
347    // Number of frames in a row that have been too old.
348    int num_consecutive_old_frames_;
349    // Number of packets in a row that have been too old.
350    int num_consecutive_old_packets_;
351    // Number of packets received.
352    int num_packets_ GUARDED_BY(crit_sect_);
353    // Number of duplicated packets received.
354    int num_duplicated_packets_ GUARDED_BY(crit_sect_);
355    // Number of packets discarded by the jitter buffer.
356    int num_discarded_packets_ GUARDED_BY(crit_sect_);
357    // Time when first packet is received.
358    int64_t time_first_packet_ms_ GUARDED_BY(crit_sect_);
359  
360    // Jitter estimation.
361    // Filter for estimating jitter.
362    VCMJitterEstimator jitter_estimate_;
363    // Calculates network delays used for jitter calculations.
364    VCMInterFrameDelay inter_frame_delay_;
365    VCMJitterSample waiting_for_completion_;
366    int64_t rtt_ms_;
367  
368    // NACK and retransmissions.
369    VCMNackMode nack_mode_;
370    int64_t low_rtt_nack_threshold_ms_;
371    int64_t high_rtt_nack_threshold_ms_;
372    // Holds the internal NACK list (the missing sequence numbers).
373    SequenceNumberSet missing_sequence_numbers_;
374    uint16_t latest_received_sequence_number_;
375    size_t max_nack_list_size_;
376    int max_packet_age_to_nack_;  // Measured in sequence numbers.
377    int max_incomplete_time_ms_;
378  
379    VCMDecodeErrorMode decode_error_mode_;
380    // Estimated rolling average of packets per frame
381    float average_packets_per_frame_;
382    // average_packets_per_frame converges fast if we have fewer than this many
383    // frames.
384    int frame_counter_;
385    RTC_DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer);
386  };
387  }  // namespace webrtc
388  
389  #endif  // WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
390