1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include "webrtc/modules/video_coding/jitter_buffer.h"
11 
12 #include <assert.h>
13 
14 #include <algorithm>
15 #include <utility>
16 
17 #include "webrtc/base/checks.h"
18 #include "webrtc/base/logging.h"
19 #include "webrtc/base/trace_event.h"
20 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
21 #include "webrtc/modules/video_coding/include/video_coding.h"
22 #include "webrtc/modules/video_coding/frame_buffer.h"
23 #include "webrtc/modules/video_coding/inter_frame_delay.h"
24 #include "webrtc/modules/video_coding/internal_defines.h"
25 #include "webrtc/modules/video_coding/jitter_buffer_common.h"
26 #include "webrtc/modules/video_coding/jitter_estimator.h"
27 #include "webrtc/modules/video_coding/packet.h"
28 #include "webrtc/system_wrappers/include/clock.h"
29 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
30 #include "webrtc/system_wrappers/include/event_wrapper.h"
31 #include "webrtc/system_wrappers/include/metrics.h"
32 
33 namespace webrtc {
34 
35 // Interval for updating SS data.
36 static const uint32_t kSsCleanupIntervalSec = 60;
37 
38 // Use this rtt if no value has been reported.
39 static const int64_t kDefaultRtt = 200;
40 
41 // Request a keyframe if no continuous frame has been received for this
42 // number of milliseconds and NACKs are disabled.
43 static const int64_t kMaxDiscontinuousFramesTime = 1000;
44 
45 typedef std::pair<uint32_t, VCMFrameBuffer*> FrameListPair;
46 
IsKeyFrame(FrameListPair pair)47 bool IsKeyFrame(FrameListPair pair) {
48   return pair.second->FrameType() == kVideoFrameKey;
49 }
50 
HasNonEmptyState(FrameListPair pair)51 bool HasNonEmptyState(FrameListPair pair) {
52   return pair.second->GetState() != kStateEmpty;
53 }
54 
InsertFrame(VCMFrameBuffer * frame)55 void FrameList::InsertFrame(VCMFrameBuffer* frame) {
56   insert(rbegin().base(), FrameListPair(frame->TimeStamp(), frame));
57 }
58 
PopFrame(uint32_t timestamp)59 VCMFrameBuffer* FrameList::PopFrame(uint32_t timestamp) {
60   FrameList::iterator it = find(timestamp);
61   if (it == end())
62     return NULL;
63   VCMFrameBuffer* frame = it->second;
64   erase(it);
65   return frame;
66 }
67 
Front() const68 VCMFrameBuffer* FrameList::Front() const {
69   return begin()->second;
70 }
71 
Back() const72 VCMFrameBuffer* FrameList::Back() const {
73   return rbegin()->second;
74 }
75 
RecycleFramesUntilKeyFrame(FrameList::iterator * key_frame_it,UnorderedFrameList * free_frames)76 int FrameList::RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
77                                           UnorderedFrameList* free_frames) {
78   int drop_count = 0;
79   FrameList::iterator it = begin();
80   while (!empty()) {
81     // Throw at least one frame.
82     it->second->Reset();
83     free_frames->push_back(it->second);
84     erase(it++);
85     ++drop_count;
86     if (it != end() && it->second->FrameType() == kVideoFrameKey) {
87       *key_frame_it = it;
88       return drop_count;
89     }
90   }
91   *key_frame_it = end();
92   return drop_count;
93 }
94 
CleanUpOldOrEmptyFrames(VCMDecodingState * decoding_state,UnorderedFrameList * free_frames)95 void FrameList::CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
96                                         UnorderedFrameList* free_frames) {
97   while (!empty()) {
98     VCMFrameBuffer* oldest_frame = Front();
99     bool remove_frame = false;
100     if (oldest_frame->GetState() == kStateEmpty && size() > 1) {
101       // This frame is empty, try to update the last decoded state and drop it
102       // if successful.
103       remove_frame = decoding_state->UpdateEmptyFrame(oldest_frame);
104     } else {
105       remove_frame = decoding_state->IsOldFrame(oldest_frame);
106     }
107     if (!remove_frame) {
108       break;
109     }
110     free_frames->push_back(oldest_frame);
111     TRACE_EVENT_INSTANT1("webrtc", "JB::OldOrEmptyFrameDropped", "timestamp",
112                          oldest_frame->TimeStamp());
113     erase(begin());
114   }
115 }
116 
Reset(UnorderedFrameList * free_frames)117 void FrameList::Reset(UnorderedFrameList* free_frames) {
118   while (!empty()) {
119     begin()->second->Reset();
120     free_frames->push_back(begin()->second);
121     erase(begin());
122   }
123 }
124 
Insert(const VCMPacket & packet)125 bool Vp9SsMap::Insert(const VCMPacket& packet) {
126   if (!packet.codecSpecificHeader.codecHeader.VP9.ss_data_available)
127     return false;
128 
129   ss_map_[packet.timestamp] = packet.codecSpecificHeader.codecHeader.VP9.gof;
130   return true;
131 }
132 
Reset()133 void Vp9SsMap::Reset() {
134   ss_map_.clear();
135 }
136 
Find(uint32_t timestamp,SsMap::iterator * it_out)137 bool Vp9SsMap::Find(uint32_t timestamp, SsMap::iterator* it_out) {
138   bool found = false;
139   for (SsMap::iterator it = ss_map_.begin(); it != ss_map_.end(); ++it) {
140     if (it->first == timestamp || IsNewerTimestamp(timestamp, it->first)) {
141       *it_out = it;
142       found = true;
143     }
144   }
145   return found;
146 }
147 
RemoveOld(uint32_t timestamp)148 void Vp9SsMap::RemoveOld(uint32_t timestamp) {
149   if (!TimeForCleanup(timestamp))
150     return;
151 
152   SsMap::iterator it;
153   if (!Find(timestamp, &it))
154     return;
155 
156   ss_map_.erase(ss_map_.begin(), it);
157   AdvanceFront(timestamp);
158 }
159 
TimeForCleanup(uint32_t timestamp) const160 bool Vp9SsMap::TimeForCleanup(uint32_t timestamp) const {
161   if (ss_map_.empty() || !IsNewerTimestamp(timestamp, ss_map_.begin()->first))
162     return false;
163 
164   uint32_t diff = timestamp - ss_map_.begin()->first;
165   return diff / kVideoPayloadTypeFrequency >= kSsCleanupIntervalSec;
166 }
167 
AdvanceFront(uint32_t timestamp)168 void Vp9SsMap::AdvanceFront(uint32_t timestamp) {
169   RTC_DCHECK(!ss_map_.empty());
170   GofInfoVP9 gof = ss_map_.begin()->second;
171   ss_map_.erase(ss_map_.begin());
172   ss_map_[timestamp] = gof;
173 }
174 
175 // TODO(asapersson): Update according to updates in RTP payload profile.
UpdatePacket(VCMPacket * packet)176 bool Vp9SsMap::UpdatePacket(VCMPacket* packet) {
177   uint8_t gof_idx = packet->codecSpecificHeader.codecHeader.VP9.gof_idx;
178   if (gof_idx == kNoGofIdx)
179     return false;  // No update needed.
180 
181   SsMap::iterator it;
182   if (!Find(packet->timestamp, &it))
183     return false;  // Corresponding SS not yet received.
184 
185   if (gof_idx >= it->second.num_frames_in_gof)
186     return false;  // Assume corresponding SS not yet received.
187 
188   RTPVideoHeaderVP9* vp9 = &packet->codecSpecificHeader.codecHeader.VP9;
189   vp9->temporal_idx = it->second.temporal_idx[gof_idx];
190   vp9->temporal_up_switch = it->second.temporal_up_switch[gof_idx];
191 
192   // TODO(asapersson): Set vp9.ref_picture_id[i] and add usage.
193   vp9->num_ref_pics = it->second.num_ref_pics[gof_idx];
194   for (uint8_t i = 0; i < it->second.num_ref_pics[gof_idx]; ++i) {
195     vp9->pid_diff[i] = it->second.pid_diff[gof_idx][i];
196   }
197   return true;
198 }
199 
UpdateFrames(FrameList * frames)200 void Vp9SsMap::UpdateFrames(FrameList* frames) {
201   for (const auto& frame_it : *frames) {
202     uint8_t gof_idx =
203         frame_it.second->CodecSpecific()->codecSpecific.VP9.gof_idx;
204     if (gof_idx == kNoGofIdx) {
205       continue;
206     }
207     SsMap::iterator ss_it;
208     if (Find(frame_it.second->TimeStamp(), &ss_it)) {
209       if (gof_idx >= ss_it->second.num_frames_in_gof) {
210         continue;  // Assume corresponding SS not yet received.
211       }
212       frame_it.second->SetGofInfo(ss_it->second, gof_idx);
213     }
214   }
215 }
216 
VCMJitterBuffer(Clock * clock,rtc::scoped_ptr<EventWrapper> event)217 VCMJitterBuffer::VCMJitterBuffer(Clock* clock,
218                                  rtc::scoped_ptr<EventWrapper> event)
219     : clock_(clock),
220       running_(false),
221       crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
222       frame_event_(std::move(event)),
223       max_number_of_frames_(kStartNumberOfFrames),
224       free_frames_(),
225       decodable_frames_(),
226       incomplete_frames_(),
227       last_decoded_state_(),
228       first_packet_since_reset_(true),
229       stats_callback_(NULL),
230       incoming_frame_rate_(0),
231       incoming_frame_count_(0),
232       time_last_incoming_frame_count_(0),
233       incoming_bit_count_(0),
234       incoming_bit_rate_(0),
235       num_consecutive_old_packets_(0),
236       num_packets_(0),
237       num_duplicated_packets_(0),
238       num_discarded_packets_(0),
239       time_first_packet_ms_(0),
240       jitter_estimate_(clock),
241       inter_frame_delay_(clock_->TimeInMilliseconds()),
242       rtt_ms_(kDefaultRtt),
243       nack_mode_(kNoNack),
244       low_rtt_nack_threshold_ms_(-1),
245       high_rtt_nack_threshold_ms_(-1),
246       missing_sequence_numbers_(SequenceNumberLessThan()),
247       max_nack_list_size_(0),
248       max_packet_age_to_nack_(0),
249       max_incomplete_time_ms_(0),
250       decode_error_mode_(kNoErrors),
251       average_packets_per_frame_(0.0f),
252       frame_counter_(0) {
253   for (int i = 0; i < kStartNumberOfFrames; i++)
254     free_frames_.push_back(new VCMFrameBuffer());
255 }
256 
~VCMJitterBuffer()257 VCMJitterBuffer::~VCMJitterBuffer() {
258   Stop();
259   for (UnorderedFrameList::iterator it = free_frames_.begin();
260        it != free_frames_.end(); ++it) {
261     delete *it;
262   }
263   for (FrameList::iterator it = incomplete_frames_.begin();
264        it != incomplete_frames_.end(); ++it) {
265     delete it->second;
266   }
267   for (FrameList::iterator it = decodable_frames_.begin();
268        it != decodable_frames_.end(); ++it) {
269     delete it->second;
270   }
271   delete crit_sect_;
272 }
273 
UpdateHistograms()274 void VCMJitterBuffer::UpdateHistograms() {
275   if (num_packets_ <= 0 || !running_) {
276     return;
277   }
278   int64_t elapsed_sec =
279       (clock_->TimeInMilliseconds() - time_first_packet_ms_) / 1000;
280   if (elapsed_sec < metrics::kMinRunTimeInSeconds) {
281     return;
282   }
283 
284   RTC_HISTOGRAM_PERCENTAGE_SPARSE("WebRTC.Video.DiscardedPacketsInPercent",
285                                   num_discarded_packets_ * 100 / num_packets_);
286   RTC_HISTOGRAM_PERCENTAGE_SPARSE("WebRTC.Video.DuplicatedPacketsInPercent",
287                                   num_duplicated_packets_ * 100 / num_packets_);
288 
289   int total_frames =
290       receive_statistics_.key_frames + receive_statistics_.delta_frames;
291   if (total_frames > 0) {
292     RTC_HISTOGRAM_COUNTS_SPARSE_100(
293         "WebRTC.Video.CompleteFramesReceivedPerSecond",
294         static_cast<int>((total_frames / elapsed_sec) + 0.5f));
295     RTC_HISTOGRAM_COUNTS_SPARSE_1000(
296         "WebRTC.Video.KeyFramesReceivedInPermille",
297         static_cast<int>(
298             (receive_statistics_.key_frames * 1000.0f / total_frames) + 0.5f));
299   }
300 }
301 
Start()302 void VCMJitterBuffer::Start() {
303   CriticalSectionScoped cs(crit_sect_);
304   running_ = true;
305   incoming_frame_count_ = 0;
306   incoming_frame_rate_ = 0;
307   incoming_bit_count_ = 0;
308   incoming_bit_rate_ = 0;
309   time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
310   receive_statistics_ = FrameCounts();
311 
312   num_consecutive_old_packets_ = 0;
313   num_packets_ = 0;
314   num_duplicated_packets_ = 0;
315   num_discarded_packets_ = 0;
316   time_first_packet_ms_ = 0;
317 
318   // Start in a non-signaled state.
319   waiting_for_completion_.frame_size = 0;
320   waiting_for_completion_.timestamp = 0;
321   waiting_for_completion_.latest_packet_time = -1;
322   first_packet_since_reset_ = true;
323   rtt_ms_ = kDefaultRtt;
324   last_decoded_state_.Reset();
325 }
326 
Stop()327 void VCMJitterBuffer::Stop() {
328   crit_sect_->Enter();
329   UpdateHistograms();
330   running_ = false;
331   last_decoded_state_.Reset();
332 
333   // Make sure all frames are free and reset.
334   for (FrameList::iterator it = decodable_frames_.begin();
335        it != decodable_frames_.end(); ++it) {
336     free_frames_.push_back(it->second);
337   }
338   for (FrameList::iterator it = incomplete_frames_.begin();
339        it != incomplete_frames_.end(); ++it) {
340     free_frames_.push_back(it->second);
341   }
342   for (UnorderedFrameList::iterator it = free_frames_.begin();
343        it != free_frames_.end(); ++it) {
344     (*it)->Reset();
345   }
346   decodable_frames_.clear();
347   incomplete_frames_.clear();
348   crit_sect_->Leave();
349   // Make sure we wake up any threads waiting on these events.
350   frame_event_->Set();
351 }
352 
Running() const353 bool VCMJitterBuffer::Running() const {
354   CriticalSectionScoped cs(crit_sect_);
355   return running_;
356 }
357 
Flush()358 void VCMJitterBuffer::Flush() {
359   CriticalSectionScoped cs(crit_sect_);
360   decodable_frames_.Reset(&free_frames_);
361   incomplete_frames_.Reset(&free_frames_);
362   last_decoded_state_.Reset();  // TODO(mikhal): sync reset.
363   num_consecutive_old_packets_ = 0;
364   // Also reset the jitter and delay estimates
365   jitter_estimate_.Reset();
366   inter_frame_delay_.Reset(clock_->TimeInMilliseconds());
367   waiting_for_completion_.frame_size = 0;
368   waiting_for_completion_.timestamp = 0;
369   waiting_for_completion_.latest_packet_time = -1;
370   first_packet_since_reset_ = true;
371   missing_sequence_numbers_.clear();
372 }
373 
374 // Get received key and delta frames
FrameStatistics() const375 FrameCounts VCMJitterBuffer::FrameStatistics() const {
376   CriticalSectionScoped cs(crit_sect_);
377   return receive_statistics_;
378 }
379 
num_packets() const380 int VCMJitterBuffer::num_packets() const {
381   CriticalSectionScoped cs(crit_sect_);
382   return num_packets_;
383 }
384 
num_duplicated_packets() const385 int VCMJitterBuffer::num_duplicated_packets() const {
386   CriticalSectionScoped cs(crit_sect_);
387   return num_duplicated_packets_;
388 }
389 
num_discarded_packets() const390 int VCMJitterBuffer::num_discarded_packets() const {
391   CriticalSectionScoped cs(crit_sect_);
392   return num_discarded_packets_;
393 }
394 
395 // Calculate framerate and bitrate.
IncomingRateStatistics(unsigned int * framerate,unsigned int * bitrate)396 void VCMJitterBuffer::IncomingRateStatistics(unsigned int* framerate,
397                                              unsigned int* bitrate) {
398   assert(framerate);
399   assert(bitrate);
400   CriticalSectionScoped cs(crit_sect_);
401   const int64_t now = clock_->TimeInMilliseconds();
402   int64_t diff = now - time_last_incoming_frame_count_;
403   if (diff < 1000 && incoming_frame_rate_ > 0 && incoming_bit_rate_ > 0) {
404     // Make sure we report something even though less than
405     // 1 second has passed since last update.
406     *framerate = incoming_frame_rate_;
407     *bitrate = incoming_bit_rate_;
408   } else if (incoming_frame_count_ != 0) {
409     // We have received frame(s) since last call to this function
410 
411     // Prepare calculations
412     if (diff <= 0) {
413       diff = 1;
414     }
415     // we add 0.5f for rounding
416     float rate = 0.5f + ((incoming_frame_count_ * 1000.0f) / diff);
417     if (rate < 1.0f) {
418       rate = 1.0f;
419     }
420 
421     // Calculate frame rate
422     // Let r be rate.
423     // r(0) = 1000*framecount/delta_time.
424     // (I.e. frames per second since last calculation.)
425     // frame_rate = r(0)/2 + r(-1)/2
426     // (I.e. fr/s average this and the previous calculation.)
427     *framerate = (incoming_frame_rate_ + static_cast<unsigned int>(rate)) / 2;
428     incoming_frame_rate_ = static_cast<unsigned int>(rate);
429 
430     // Calculate bit rate
431     if (incoming_bit_count_ == 0) {
432       *bitrate = 0;
433     } else {
434       *bitrate =
435           10 * ((100 * incoming_bit_count_) / static_cast<unsigned int>(diff));
436     }
437     incoming_bit_rate_ = *bitrate;
438 
439     // Reset count
440     incoming_frame_count_ = 0;
441     incoming_bit_count_ = 0;
442     time_last_incoming_frame_count_ = now;
443 
444   } else {
445     // No frames since last call
446     time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
447     *framerate = 0;
448     *bitrate = 0;
449     incoming_frame_rate_ = 0;
450     incoming_bit_rate_ = 0;
451   }
452 }
453 
454 // Answers the question:
455 // Will the packet sequence be complete if the next frame is grabbed for
456 // decoding right now? That is, have we lost a frame between the last decoded
457 // frame and the next, or is the next
458 // frame missing one or more packets?
CompleteSequenceWithNextFrame()459 bool VCMJitterBuffer::CompleteSequenceWithNextFrame() {
460   CriticalSectionScoped cs(crit_sect_);
461   // Finding oldest frame ready for decoder, check sequence number and size
462   CleanUpOldOrEmptyFrames();
463   if (!decodable_frames_.empty()) {
464     if (decodable_frames_.Front()->GetState() == kStateComplete) {
465       return true;
466     }
467   } else if (incomplete_frames_.size() <= 1) {
468     // Frame not ready to be decoded.
469     return true;
470   }
471   return false;
472 }
473 
474 // Returns immediately or a |max_wait_time_ms| ms event hang waiting for a
475 // complete frame, |max_wait_time_ms| decided by caller.
NextCompleteTimestamp(uint32_t max_wait_time_ms,uint32_t * timestamp)476 bool VCMJitterBuffer::NextCompleteTimestamp(uint32_t max_wait_time_ms,
477                                             uint32_t* timestamp) {
478   crit_sect_->Enter();
479   if (!running_) {
480     crit_sect_->Leave();
481     return false;
482   }
483   CleanUpOldOrEmptyFrames();
484 
485   if (decodable_frames_.empty() ||
486       decodable_frames_.Front()->GetState() != kStateComplete) {
487     const int64_t end_wait_time_ms =
488         clock_->TimeInMilliseconds() + max_wait_time_ms;
489     int64_t wait_time_ms = max_wait_time_ms;
490     while (wait_time_ms > 0) {
491       crit_sect_->Leave();
492       const EventTypeWrapper ret =
493           frame_event_->Wait(static_cast<uint32_t>(wait_time_ms));
494       crit_sect_->Enter();
495       if (ret == kEventSignaled) {
496         // Are we shutting down the jitter buffer?
497         if (!running_) {
498           crit_sect_->Leave();
499           return false;
500         }
501         // Finding oldest frame ready for decoder.
502         CleanUpOldOrEmptyFrames();
503         if (decodable_frames_.empty() ||
504             decodable_frames_.Front()->GetState() != kStateComplete) {
505           wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds();
506         } else {
507           break;
508         }
509       } else {
510         break;
511       }
512     }
513   }
514   if (decodable_frames_.empty() ||
515       decodable_frames_.Front()->GetState() != kStateComplete) {
516     crit_sect_->Leave();
517     return false;
518   }
519   *timestamp = decodable_frames_.Front()->TimeStamp();
520   crit_sect_->Leave();
521   return true;
522 }
523 
NextMaybeIncompleteTimestamp(uint32_t * timestamp)524 bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(uint32_t* timestamp) {
525   CriticalSectionScoped cs(crit_sect_);
526   if (!running_) {
527     return false;
528   }
529   if (decode_error_mode_ == kNoErrors) {
530     // No point to continue, as we are not decoding with errors.
531     return false;
532   }
533 
534   CleanUpOldOrEmptyFrames();
535 
536   VCMFrameBuffer* oldest_frame;
537   if (decodable_frames_.empty()) {
538     if (nack_mode_ != kNoNack || incomplete_frames_.size() <= 1) {
539       return false;
540     }
541     oldest_frame = incomplete_frames_.Front();
542     // Frame will only be removed from buffer if it is complete (or decodable).
543     if (oldest_frame->GetState() < kStateComplete) {
544       return false;
545     }
546   } else {
547     oldest_frame = decodable_frames_.Front();
548     // If we have exactly one frame in the buffer, release it only if it is
549     // complete. We know decodable_frames_ is  not empty due to the previous
550     // check.
551     if (decodable_frames_.size() == 1 && incomplete_frames_.empty() &&
552         oldest_frame->GetState() != kStateComplete) {
553       return false;
554     }
555   }
556 
557   *timestamp = oldest_frame->TimeStamp();
558   return true;
559 }
560 
ExtractAndSetDecode(uint32_t timestamp)561 VCMEncodedFrame* VCMJitterBuffer::ExtractAndSetDecode(uint32_t timestamp) {
562   CriticalSectionScoped cs(crit_sect_);
563   if (!running_) {
564     return NULL;
565   }
566   // Extract the frame with the desired timestamp.
567   VCMFrameBuffer* frame = decodable_frames_.PopFrame(timestamp);
568   bool continuous = true;
569   if (!frame) {
570     frame = incomplete_frames_.PopFrame(timestamp);
571     if (frame)
572       continuous = last_decoded_state_.ContinuousFrame(frame);
573     else
574       return NULL;
575   }
576   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", timestamp, "Extract");
577   // Frame pulled out from jitter buffer, update the jitter estimate.
578   const bool retransmitted = (frame->GetNackCount() > 0);
579   if (retransmitted) {
580     jitter_estimate_.FrameNacked();
581   } else if (frame->Length() > 0) {
582     // Ignore retransmitted and empty frames.
583     if (waiting_for_completion_.latest_packet_time >= 0) {
584       UpdateJitterEstimate(waiting_for_completion_, true);
585     }
586     if (frame->GetState() == kStateComplete) {
587       UpdateJitterEstimate(*frame, false);
588     } else {
589       // Wait for this one to get complete.
590       waiting_for_completion_.frame_size = frame->Length();
591       waiting_for_completion_.latest_packet_time = frame->LatestPacketTimeMs();
592       waiting_for_completion_.timestamp = frame->TimeStamp();
593     }
594   }
595 
596   // The state must be changed to decoding before cleaning up zero sized
597   // frames to avoid empty frames being cleaned up and then given to the
598   // decoder. Propagates the missing_frame bit.
599   frame->PrepareForDecode(continuous);
600 
601   // We have a frame - update the last decoded state and nack list.
602   last_decoded_state_.SetState(frame);
603   DropPacketsFromNackList(last_decoded_state_.sequence_num());
604 
605   if ((*frame).IsSessionComplete())
606     UpdateAveragePacketsPerFrame(frame->NumPackets());
607 
608   return frame;
609 }
610 
611 // Release frame when done with decoding. Should never be used to release
612 // frames from within the jitter buffer.
ReleaseFrame(VCMEncodedFrame * frame)613 void VCMJitterBuffer::ReleaseFrame(VCMEncodedFrame* frame) {
614   CriticalSectionScoped cs(crit_sect_);
615   VCMFrameBuffer* frame_buffer = static_cast<VCMFrameBuffer*>(frame);
616   if (frame_buffer) {
617     free_frames_.push_back(frame_buffer);
618   }
619 }
620 
621 // Gets frame to use for this timestamp. If no match, get empty frame.
GetFrame(const VCMPacket & packet,VCMFrameBuffer ** frame,FrameList ** frame_list)622 VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet,
623                                              VCMFrameBuffer** frame,
624                                              FrameList** frame_list) {
625   *frame = incomplete_frames_.PopFrame(packet.timestamp);
626   if (*frame != NULL) {
627     *frame_list = &incomplete_frames_;
628     return kNoError;
629   }
630   *frame = decodable_frames_.PopFrame(packet.timestamp);
631   if (*frame != NULL) {
632     *frame_list = &decodable_frames_;
633     return kNoError;
634   }
635 
636   *frame_list = NULL;
637   // No match, return empty frame.
638   *frame = GetEmptyFrame();
639   if (*frame == NULL) {
640     // No free frame! Try to reclaim some...
641     LOG(LS_WARNING) << "Unable to get empty frame; Recycling.";
642     bool found_key_frame = RecycleFramesUntilKeyFrame();
643     *frame = GetEmptyFrame();
644     assert(*frame);
645     if (!found_key_frame) {
646       free_frames_.push_back(*frame);
647       return kFlushIndicator;
648     }
649   }
650   (*frame)->Reset();
651   return kNoError;
652 }
653 
LastPacketTime(const VCMEncodedFrame * frame,bool * retransmitted) const654 int64_t VCMJitterBuffer::LastPacketTime(const VCMEncodedFrame* frame,
655                                         bool* retransmitted) const {
656   assert(retransmitted);
657   CriticalSectionScoped cs(crit_sect_);
658   const VCMFrameBuffer* frame_buffer =
659       static_cast<const VCMFrameBuffer*>(frame);
660   *retransmitted = (frame_buffer->GetNackCount() > 0);
661   return frame_buffer->LatestPacketTimeMs();
662 }
663 
InsertPacket(const VCMPacket & packet,bool * retransmitted)664 VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
665                                                  bool* retransmitted) {
666   CriticalSectionScoped cs(crit_sect_);
667 
668   ++num_packets_;
669   if (num_packets_ == 1) {
670     time_first_packet_ms_ = clock_->TimeInMilliseconds();
671   }
672   // Does this packet belong to an old frame?
673   if (last_decoded_state_.IsOldPacket(&packet)) {
674     // Account only for media packets.
675     if (packet.sizeBytes > 0) {
676       num_discarded_packets_++;
677       num_consecutive_old_packets_++;
678       if (stats_callback_ != NULL)
679         stats_callback_->OnDiscardedPacketsUpdated(num_discarded_packets_);
680     }
681     // Update last decoded sequence number if the packet arrived late and
682     // belongs to a frame with a timestamp equal to the last decoded
683     // timestamp.
684     last_decoded_state_.UpdateOldPacket(&packet);
685     DropPacketsFromNackList(last_decoded_state_.sequence_num());
686 
687     // Also see if this old packet made more incomplete frames continuous.
688     FindAndInsertContinuousFramesWithState(last_decoded_state_);
689 
690     if (num_consecutive_old_packets_ > kMaxConsecutiveOldPackets) {
691       LOG(LS_WARNING)
692           << num_consecutive_old_packets_
693           << " consecutive old packets received. Flushing the jitter buffer.";
694       Flush();
695       return kFlushIndicator;
696     }
697     return kOldPacket;
698   }
699 
700   num_consecutive_old_packets_ = 0;
701 
702   VCMFrameBuffer* frame;
703   FrameList* frame_list;
704   const VCMFrameBufferEnum error = GetFrame(packet, &frame, &frame_list);
705   if (error != kNoError)
706     return error;
707 
708   int64_t now_ms = clock_->TimeInMilliseconds();
709   // We are keeping track of the first and latest seq numbers, and
710   // the number of wraps to be able to calculate how many packets we expect.
711   if (first_packet_since_reset_) {
712     // Now it's time to start estimating jitter
713     // reset the delay estimate.
714     inter_frame_delay_.Reset(now_ms);
715   }
716 
717   // Empty packets may bias the jitter estimate (lacking size component),
718   // therefore don't let empty packet trigger the following updates:
719   if (packet.frameType != kEmptyFrame) {
720     if (waiting_for_completion_.timestamp == packet.timestamp) {
721       // This can get bad if we have a lot of duplicate packets,
722       // we will then count some packet multiple times.
723       waiting_for_completion_.frame_size += packet.sizeBytes;
724       waiting_for_completion_.latest_packet_time = now_ms;
725     } else if (waiting_for_completion_.latest_packet_time >= 0 &&
726                waiting_for_completion_.latest_packet_time + 2000 <= now_ms) {
727       // A packet should never be more than two seconds late
728       UpdateJitterEstimate(waiting_for_completion_, true);
729       waiting_for_completion_.latest_packet_time = -1;
730       waiting_for_completion_.frame_size = 0;
731       waiting_for_completion_.timestamp = 0;
732     }
733   }
734 
735   VCMFrameBufferStateEnum previous_state = frame->GetState();
736   // Insert packet.
737   FrameData frame_data;
738   frame_data.rtt_ms = rtt_ms_;
739   frame_data.rolling_average_packets_per_frame = average_packets_per_frame_;
740   VCMFrameBufferEnum buffer_state =
741       frame->InsertPacket(packet, now_ms, decode_error_mode_, frame_data);
742 
743   if (previous_state != kStateComplete) {
744     TRACE_EVENT_ASYNC_BEGIN1("webrtc", "Video", frame->TimeStamp(), "timestamp",
745                              frame->TimeStamp());
746   }
747 
748   if (buffer_state > 0) {
749     incoming_bit_count_ += packet.sizeBytes << 3;
750     if (first_packet_since_reset_) {
751       latest_received_sequence_number_ = packet.seqNum;
752       first_packet_since_reset_ = false;
753     } else {
754       if (IsPacketRetransmitted(packet)) {
755         frame->IncrementNackCount();
756       }
757       if (!UpdateNackList(packet.seqNum) &&
758           packet.frameType != kVideoFrameKey) {
759         buffer_state = kFlushIndicator;
760       }
761 
762       latest_received_sequence_number_ =
763           LatestSequenceNumber(latest_received_sequence_number_, packet.seqNum);
764     }
765   }
766 
767   // Is the frame already in the decodable list?
768   bool continuous = IsContinuous(*frame);
769   switch (buffer_state) {
770     case kGeneralError:
771     case kTimeStampError:
772     case kSizeError: {
773       free_frames_.push_back(frame);
774       break;
775     }
776     case kCompleteSession: {
777       if (previous_state != kStateDecodable &&
778           previous_state != kStateComplete) {
779         CountFrame(*frame);
780         if (continuous) {
781           // Signal that we have a complete session.
782           frame_event_->Set();
783         }
784       }
785       FALLTHROUGH();
786     }
787     // Note: There is no break here - continuing to kDecodableSession.
788     case kDecodableSession: {
789       *retransmitted = (frame->GetNackCount() > 0);
790       if (continuous) {
791         decodable_frames_.InsertFrame(frame);
792         FindAndInsertContinuousFrames(*frame);
793       } else {
794         incomplete_frames_.InsertFrame(frame);
795         // If NACKs are enabled, keyframes are triggered by |GetNackList|.
796         if (nack_mode_ == kNoNack &&
797             NonContinuousOrIncompleteDuration() >
798                 90 * kMaxDiscontinuousFramesTime) {
799           return kFlushIndicator;
800         }
801       }
802       break;
803     }
804     case kIncomplete: {
805       if (frame->GetState() == kStateEmpty &&
806           last_decoded_state_.UpdateEmptyFrame(frame)) {
807         free_frames_.push_back(frame);
808         return kNoError;
809       } else {
810         incomplete_frames_.InsertFrame(frame);
811         // If NACKs are enabled, keyframes are triggered by |GetNackList|.
812         if (nack_mode_ == kNoNack &&
813             NonContinuousOrIncompleteDuration() >
814                 90 * kMaxDiscontinuousFramesTime) {
815           return kFlushIndicator;
816         }
817       }
818       break;
819     }
820     case kNoError:
821     case kOutOfBoundsPacket:
822     case kDuplicatePacket: {
823       // Put back the frame where it came from.
824       if (frame_list != NULL) {
825         frame_list->InsertFrame(frame);
826       } else {
827         free_frames_.push_back(frame);
828       }
829       ++num_duplicated_packets_;
830       break;
831     }
832     case kFlushIndicator:
833       free_frames_.push_back(frame);
834       return kFlushIndicator;
835     default:
836       assert(false);
837   }
838   return buffer_state;
839 }
840 
IsContinuousInState(const VCMFrameBuffer & frame,const VCMDecodingState & decoding_state) const841 bool VCMJitterBuffer::IsContinuousInState(
842     const VCMFrameBuffer& frame,
843     const VCMDecodingState& decoding_state) const {
844   // Is this frame (complete or decodable) and continuous?
845   // kStateDecodable will never be set when decode_error_mode_ is false
846   // as SessionInfo determines this state based on the error mode (and frame
847   // completeness).
848   return (frame.GetState() == kStateComplete ||
849           frame.GetState() == kStateDecodable) &&
850          decoding_state.ContinuousFrame(&frame);
851 }
852 
IsContinuous(const VCMFrameBuffer & frame) const853 bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const {
854   if (IsContinuousInState(frame, last_decoded_state_)) {
855     return true;
856   }
857   VCMDecodingState decoding_state;
858   decoding_state.CopyFrom(last_decoded_state_);
859   for (FrameList::const_iterator it = decodable_frames_.begin();
860        it != decodable_frames_.end(); ++it) {
861     VCMFrameBuffer* decodable_frame = it->second;
862     if (IsNewerTimestamp(decodable_frame->TimeStamp(), frame.TimeStamp())) {
863       break;
864     }
865     decoding_state.SetState(decodable_frame);
866     if (IsContinuousInState(frame, decoding_state)) {
867       return true;
868     }
869   }
870   return false;
871 }
872 
FindAndInsertContinuousFrames(const VCMFrameBuffer & new_frame)873 void VCMJitterBuffer::FindAndInsertContinuousFrames(
874     const VCMFrameBuffer& new_frame) {
875   VCMDecodingState decoding_state;
876   decoding_state.CopyFrom(last_decoded_state_);
877   decoding_state.SetState(&new_frame);
878   FindAndInsertContinuousFramesWithState(decoding_state);
879 }
880 
FindAndInsertContinuousFramesWithState(const VCMDecodingState & original_decoded_state)881 void VCMJitterBuffer::FindAndInsertContinuousFramesWithState(
882     const VCMDecodingState& original_decoded_state) {
883   // Copy original_decoded_state so we can move the state forward with each
884   // decodable frame we find.
885   VCMDecodingState decoding_state;
886   decoding_state.CopyFrom(original_decoded_state);
887 
888   // When temporal layers are available, we search for a complete or decodable
889   // frame until we hit one of the following:
890   // 1. Continuous base or sync layer.
891   // 2. The end of the list was reached.
892   for (FrameList::iterator it = incomplete_frames_.begin();
893        it != incomplete_frames_.end();) {
894     VCMFrameBuffer* frame = it->second;
895     if (IsNewerTimestamp(original_decoded_state.time_stamp(),
896                          frame->TimeStamp())) {
897       ++it;
898       continue;
899     }
900     if (IsContinuousInState(*frame, decoding_state)) {
901       decodable_frames_.InsertFrame(frame);
902       incomplete_frames_.erase(it++);
903       decoding_state.SetState(frame);
904     } else if (frame->TemporalId() <= 0) {
905       break;
906     } else {
907       ++it;
908     }
909   }
910 }
911 
EstimatedJitterMs()912 uint32_t VCMJitterBuffer::EstimatedJitterMs() {
913   CriticalSectionScoped cs(crit_sect_);
914   // Compute RTT multiplier for estimation.
915   // low_rtt_nackThresholdMs_ == -1 means no FEC.
916   double rtt_mult = 1.0f;
917   if (low_rtt_nack_threshold_ms_ >= 0 &&
918       rtt_ms_ >= low_rtt_nack_threshold_ms_) {
919     // For RTTs above low_rtt_nack_threshold_ms_ we don't apply extra delay
920     // when waiting for retransmissions.
921     rtt_mult = 0.0f;
922   }
923   return jitter_estimate_.GetJitterEstimate(rtt_mult);
924 }
925 
UpdateRtt(int64_t rtt_ms)926 void VCMJitterBuffer::UpdateRtt(int64_t rtt_ms) {
927   CriticalSectionScoped cs(crit_sect_);
928   rtt_ms_ = rtt_ms;
929   jitter_estimate_.UpdateRtt(rtt_ms);
930 }
931 
SetNackMode(VCMNackMode mode,int64_t low_rtt_nack_threshold_ms,int64_t high_rtt_nack_threshold_ms)932 void VCMJitterBuffer::SetNackMode(VCMNackMode mode,
933                                   int64_t low_rtt_nack_threshold_ms,
934                                   int64_t high_rtt_nack_threshold_ms) {
935   CriticalSectionScoped cs(crit_sect_);
936   nack_mode_ = mode;
937   if (mode == kNoNack) {
938     missing_sequence_numbers_.clear();
939   }
940   assert(low_rtt_nack_threshold_ms >= -1 && high_rtt_nack_threshold_ms >= -1);
941   assert(high_rtt_nack_threshold_ms == -1 ||
942          low_rtt_nack_threshold_ms <= high_rtt_nack_threshold_ms);
943   assert(low_rtt_nack_threshold_ms > -1 || high_rtt_nack_threshold_ms == -1);
944   low_rtt_nack_threshold_ms_ = low_rtt_nack_threshold_ms;
945   high_rtt_nack_threshold_ms_ = high_rtt_nack_threshold_ms;
946   // Don't set a high start rtt if high_rtt_nack_threshold_ms_ is used, to not
947   // disable NACK in |kNack| mode.
948   if (rtt_ms_ == kDefaultRtt && high_rtt_nack_threshold_ms_ != -1) {
949     rtt_ms_ = 0;
950   }
951   if (!WaitForRetransmissions()) {
952     jitter_estimate_.ResetNackCount();
953   }
954 }
955 
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)956 void VCMJitterBuffer::SetNackSettings(size_t max_nack_list_size,
957                                       int max_packet_age_to_nack,
958                                       int max_incomplete_time_ms) {
959   CriticalSectionScoped cs(crit_sect_);
960   assert(max_packet_age_to_nack >= 0);
961   assert(max_incomplete_time_ms_ >= 0);
962   max_nack_list_size_ = max_nack_list_size;
963   max_packet_age_to_nack_ = max_packet_age_to_nack;
964   max_incomplete_time_ms_ = max_incomplete_time_ms;
965 }
966 
nack_mode() const967 VCMNackMode VCMJitterBuffer::nack_mode() const {
968   CriticalSectionScoped cs(crit_sect_);
969   return nack_mode_;
970 }
971 
NonContinuousOrIncompleteDuration()972 int VCMJitterBuffer::NonContinuousOrIncompleteDuration() {
973   if (incomplete_frames_.empty()) {
974     return 0;
975   }
976   uint32_t start_timestamp = incomplete_frames_.Front()->TimeStamp();
977   if (!decodable_frames_.empty()) {
978     start_timestamp = decodable_frames_.Back()->TimeStamp();
979   }
980   return incomplete_frames_.Back()->TimeStamp() - start_timestamp;
981 }
982 
EstimatedLowSequenceNumber(const VCMFrameBuffer & frame) const983 uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber(
984     const VCMFrameBuffer& frame) const {
985   assert(frame.GetLowSeqNum() >= 0);
986   if (frame.HaveFirstPacket())
987     return frame.GetLowSeqNum();
988 
989   // This estimate is not accurate if more than one packet with lower sequence
990   // number is lost.
991   return frame.GetLowSeqNum() - 1;
992 }
993 
GetNackList(bool * request_key_frame)994 std::vector<uint16_t> VCMJitterBuffer::GetNackList(bool* request_key_frame) {
995   CriticalSectionScoped cs(crit_sect_);
996   *request_key_frame = false;
997   if (nack_mode_ == kNoNack) {
998     return std::vector<uint16_t>();
999   }
1000   if (last_decoded_state_.in_initial_state()) {
1001     VCMFrameBuffer* next_frame = NextFrame();
1002     const bool first_frame_is_key = next_frame &&
1003                                     next_frame->FrameType() == kVideoFrameKey &&
1004                                     next_frame->HaveFirstPacket();
1005     if (!first_frame_is_key) {
1006       bool have_non_empty_frame =
1007           decodable_frames_.end() != find_if(decodable_frames_.begin(),
1008                                              decodable_frames_.end(),
1009                                              HasNonEmptyState);
1010       if (!have_non_empty_frame) {
1011         have_non_empty_frame =
1012             incomplete_frames_.end() != find_if(incomplete_frames_.begin(),
1013                                                 incomplete_frames_.end(),
1014                                                 HasNonEmptyState);
1015       }
1016       bool found_key_frame = RecycleFramesUntilKeyFrame();
1017       if (!found_key_frame) {
1018         *request_key_frame = have_non_empty_frame;
1019         return std::vector<uint16_t>();
1020       }
1021     }
1022   }
1023   if (TooLargeNackList()) {
1024     *request_key_frame = !HandleTooLargeNackList();
1025   }
1026   if (max_incomplete_time_ms_ > 0) {
1027     int non_continuous_incomplete_duration =
1028         NonContinuousOrIncompleteDuration();
1029     if (non_continuous_incomplete_duration > 90 * max_incomplete_time_ms_) {
1030       LOG_F(LS_WARNING) << "Too long non-decodable duration: "
1031                         << non_continuous_incomplete_duration << " > "
1032                         << 90 * max_incomplete_time_ms_;
1033       FrameList::reverse_iterator rit = find_if(
1034           incomplete_frames_.rbegin(), incomplete_frames_.rend(), IsKeyFrame);
1035       if (rit == incomplete_frames_.rend()) {
1036         // Request a key frame if we don't have one already.
1037         *request_key_frame = true;
1038         return std::vector<uint16_t>();
1039       } else {
1040         // Skip to the last key frame. If it's incomplete we will start
1041         // NACKing it.
1042         // Note that the estimated low sequence number is correct for VP8
1043         // streams because only the first packet of a key frame is marked.
1044         last_decoded_state_.Reset();
1045         DropPacketsFromNackList(EstimatedLowSequenceNumber(*rit->second));
1046       }
1047     }
1048   }
1049   std::vector<uint16_t> nack_list(missing_sequence_numbers_.begin(),
1050                                   missing_sequence_numbers_.end());
1051   return nack_list;
1052 }
1053 
SetDecodeErrorMode(VCMDecodeErrorMode error_mode)1054 void VCMJitterBuffer::SetDecodeErrorMode(VCMDecodeErrorMode error_mode) {
1055   CriticalSectionScoped cs(crit_sect_);
1056   decode_error_mode_ = error_mode;
1057 }
1058 
NextFrame() const1059 VCMFrameBuffer* VCMJitterBuffer::NextFrame() const {
1060   if (!decodable_frames_.empty())
1061     return decodable_frames_.Front();
1062   if (!incomplete_frames_.empty())
1063     return incomplete_frames_.Front();
1064   return NULL;
1065 }
1066 
UpdateNackList(uint16_t sequence_number)1067 bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) {
1068   if (nack_mode_ == kNoNack) {
1069     return true;
1070   }
1071   // Make sure we don't add packets which are already too old to be decoded.
1072   if (!last_decoded_state_.in_initial_state()) {
1073     latest_received_sequence_number_ = LatestSequenceNumber(
1074         latest_received_sequence_number_, last_decoded_state_.sequence_num());
1075   }
1076   if (IsNewerSequenceNumber(sequence_number,
1077                             latest_received_sequence_number_)) {
1078     // Push any missing sequence numbers to the NACK list.
1079     for (uint16_t i = latest_received_sequence_number_ + 1;
1080          IsNewerSequenceNumber(sequence_number, i); ++i) {
1081       missing_sequence_numbers_.insert(missing_sequence_numbers_.end(), i);
1082       TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), "AddNack",
1083                            "seqnum", i);
1084     }
1085     if (TooLargeNackList() && !HandleTooLargeNackList()) {
1086       LOG(LS_WARNING) << "Requesting key frame due to too large NACK list.";
1087       return false;
1088     }
1089     if (MissingTooOldPacket(sequence_number) &&
1090         !HandleTooOldPackets(sequence_number)) {
1091       LOG(LS_WARNING) << "Requesting key frame due to missing too old packets";
1092       return false;
1093     }
1094   } else {
1095     missing_sequence_numbers_.erase(sequence_number);
1096     TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), "RemoveNack",
1097                          "seqnum", sequence_number);
1098   }
1099   return true;
1100 }
1101 
TooLargeNackList() const1102 bool VCMJitterBuffer::TooLargeNackList() const {
1103   return missing_sequence_numbers_.size() > max_nack_list_size_;
1104 }
1105 
HandleTooLargeNackList()1106 bool VCMJitterBuffer::HandleTooLargeNackList() {
1107   // Recycle frames until the NACK list is small enough. It is likely cheaper to
1108   // request a key frame than to retransmit this many missing packets.
1109   LOG_F(LS_WARNING) << "NACK list has grown too large: "
1110                     << missing_sequence_numbers_.size() << " > "
1111                     << max_nack_list_size_;
1112   bool key_frame_found = false;
1113   while (TooLargeNackList()) {
1114     key_frame_found = RecycleFramesUntilKeyFrame();
1115   }
1116   return key_frame_found;
1117 }
1118 
MissingTooOldPacket(uint16_t latest_sequence_number) const1119 bool VCMJitterBuffer::MissingTooOldPacket(
1120     uint16_t latest_sequence_number) const {
1121   if (missing_sequence_numbers_.empty()) {
1122     return false;
1123   }
1124   const uint16_t age_of_oldest_missing_packet =
1125       latest_sequence_number - *missing_sequence_numbers_.begin();
1126   // Recycle frames if the NACK list contains too old sequence numbers as
1127   // the packets may have already been dropped by the sender.
1128   return age_of_oldest_missing_packet > max_packet_age_to_nack_;
1129 }
1130 
HandleTooOldPackets(uint16_t latest_sequence_number)1131 bool VCMJitterBuffer::HandleTooOldPackets(uint16_t latest_sequence_number) {
1132   bool key_frame_found = false;
1133   const uint16_t age_of_oldest_missing_packet =
1134       latest_sequence_number - *missing_sequence_numbers_.begin();
1135   LOG_F(LS_WARNING) << "NACK list contains too old sequence numbers: "
1136                     << age_of_oldest_missing_packet << " > "
1137                     << max_packet_age_to_nack_;
1138   while (MissingTooOldPacket(latest_sequence_number)) {
1139     key_frame_found = RecycleFramesUntilKeyFrame();
1140   }
1141   return key_frame_found;
1142 }
1143 
DropPacketsFromNackList(uint16_t last_decoded_sequence_number)1144 void VCMJitterBuffer::DropPacketsFromNackList(
1145     uint16_t last_decoded_sequence_number) {
1146   // Erase all sequence numbers from the NACK list which we won't need any
1147   // longer.
1148   missing_sequence_numbers_.erase(
1149       missing_sequence_numbers_.begin(),
1150       missing_sequence_numbers_.upper_bound(last_decoded_sequence_number));
1151 }
1152 
LastDecodedTimestamp() const1153 int64_t VCMJitterBuffer::LastDecodedTimestamp() const {
1154   CriticalSectionScoped cs(crit_sect_);
1155   return last_decoded_state_.time_stamp();
1156 }
1157 
RenderBufferSize(uint32_t * timestamp_start,uint32_t * timestamp_end)1158 void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start,
1159                                        uint32_t* timestamp_end) {
1160   CriticalSectionScoped cs(crit_sect_);
1161   CleanUpOldOrEmptyFrames();
1162   *timestamp_start = 0;
1163   *timestamp_end = 0;
1164   if (decodable_frames_.empty()) {
1165     return;
1166   }
1167   *timestamp_start = decodable_frames_.Front()->TimeStamp();
1168   *timestamp_end = decodable_frames_.Back()->TimeStamp();
1169 }
1170 
RegisterStatsCallback(VCMReceiveStatisticsCallback * callback)1171 void VCMJitterBuffer::RegisterStatsCallback(
1172     VCMReceiveStatisticsCallback* callback) {
1173   CriticalSectionScoped cs(crit_sect_);
1174   stats_callback_ = callback;
1175 }
1176 
GetEmptyFrame()1177 VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() {
1178   if (free_frames_.empty()) {
1179     if (!TryToIncreaseJitterBufferSize()) {
1180       return NULL;
1181     }
1182   }
1183   VCMFrameBuffer* frame = free_frames_.front();
1184   free_frames_.pop_front();
1185   return frame;
1186 }
1187 
TryToIncreaseJitterBufferSize()1188 bool VCMJitterBuffer::TryToIncreaseJitterBufferSize() {
1189   if (max_number_of_frames_ >= kMaxNumberOfFrames)
1190     return false;
1191   free_frames_.push_back(new VCMFrameBuffer());
1192   ++max_number_of_frames_;
1193   TRACE_COUNTER1("webrtc", "JBMaxFrames", max_number_of_frames_);
1194   return true;
1195 }
1196 
1197 // Recycle oldest frames up to a key frame, used if jitter buffer is completely
1198 // full.
RecycleFramesUntilKeyFrame()1199 bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() {
1200   // First release incomplete frames, and only release decodable frames if there
1201   // are no incomplete ones.
1202   FrameList::iterator key_frame_it;
1203   bool key_frame_found = false;
1204   int dropped_frames = 0;
1205   dropped_frames += incomplete_frames_.RecycleFramesUntilKeyFrame(
1206       &key_frame_it, &free_frames_);
1207   key_frame_found = key_frame_it != incomplete_frames_.end();
1208   if (dropped_frames == 0) {
1209     dropped_frames += decodable_frames_.RecycleFramesUntilKeyFrame(
1210         &key_frame_it, &free_frames_);
1211     key_frame_found = key_frame_it != decodable_frames_.end();
1212   }
1213   TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame");
1214   if (key_frame_found) {
1215     LOG(LS_INFO) << "Found key frame while dropping frames.";
1216     // Reset last decoded state to make sure the next frame decoded is a key
1217     // frame, and start NACKing from here.
1218     last_decoded_state_.Reset();
1219     DropPacketsFromNackList(EstimatedLowSequenceNumber(*key_frame_it->second));
1220   } else if (decodable_frames_.empty()) {
1221     // All frames dropped. Reset the decoding state and clear missing sequence
1222     // numbers as we're starting fresh.
1223     last_decoded_state_.Reset();
1224     missing_sequence_numbers_.clear();
1225   }
1226   return key_frame_found;
1227 }
1228 
1229 // Must be called under the critical section |crit_sect_|.
CountFrame(const VCMFrameBuffer & frame)1230 void VCMJitterBuffer::CountFrame(const VCMFrameBuffer& frame) {
1231   incoming_frame_count_++;
1232 
1233   if (frame.FrameType() == kVideoFrameKey) {
1234     TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", frame.TimeStamp(),
1235                             "KeyComplete");
1236   } else {
1237     TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", frame.TimeStamp(),
1238                             "DeltaComplete");
1239   }
1240 
1241   // Update receive statistics. We count all layers, thus when you use layers
1242   // adding all key and delta frames might differ from frame count.
1243   if (frame.IsSessionComplete()) {
1244     if (frame.FrameType() == kVideoFrameKey) {
1245       ++receive_statistics_.key_frames;
1246     } else {
1247       ++receive_statistics_.delta_frames;
1248     }
1249     if (stats_callback_ != NULL)
1250       stats_callback_->OnFrameCountsUpdated(receive_statistics_);
1251   }
1252 }
1253 
UpdateAveragePacketsPerFrame(int current_number_packets)1254 void VCMJitterBuffer::UpdateAveragePacketsPerFrame(int current_number_packets) {
1255   if (frame_counter_ > kFastConvergeThreshold) {
1256     average_packets_per_frame_ =
1257         average_packets_per_frame_ * (1 - kNormalConvergeMultiplier) +
1258         current_number_packets * kNormalConvergeMultiplier;
1259   } else if (frame_counter_ > 0) {
1260     average_packets_per_frame_ =
1261         average_packets_per_frame_ * (1 - kFastConvergeMultiplier) +
1262         current_number_packets * kFastConvergeMultiplier;
1263     frame_counter_++;
1264   } else {
1265     average_packets_per_frame_ = current_number_packets;
1266     frame_counter_++;
1267   }
1268 }
1269 
1270 // Must be called under the critical section |crit_sect_|.
CleanUpOldOrEmptyFrames()1271 void VCMJitterBuffer::CleanUpOldOrEmptyFrames() {
1272   decodable_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1273                                             &free_frames_);
1274   incomplete_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1275                                              &free_frames_);
1276   if (!last_decoded_state_.in_initial_state()) {
1277     DropPacketsFromNackList(last_decoded_state_.sequence_num());
1278   }
1279 }
1280 
1281 // Must be called from within |crit_sect_|.
IsPacketRetransmitted(const VCMPacket & packet) const1282 bool VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const {
1283   return missing_sequence_numbers_.find(packet.seqNum) !=
1284          missing_sequence_numbers_.end();
1285 }
1286 
1287 // Must be called under the critical section |crit_sect_|. Should never be
1288 // called with retransmitted frames, they must be filtered out before this
1289 // function is called.
UpdateJitterEstimate(const VCMJitterSample & sample,bool incomplete_frame)1290 void VCMJitterBuffer::UpdateJitterEstimate(const VCMJitterSample& sample,
1291                                            bool incomplete_frame) {
1292   if (sample.latest_packet_time == -1) {
1293     return;
1294   }
1295   UpdateJitterEstimate(sample.latest_packet_time, sample.timestamp,
1296                        sample.frame_size, incomplete_frame);
1297 }
1298 
1299 // Must be called under the critical section crit_sect_. Should never be
1300 // called with retransmitted frames, they must be filtered out before this
1301 // function is called.
UpdateJitterEstimate(const VCMFrameBuffer & frame,bool incomplete_frame)1302 void VCMJitterBuffer::UpdateJitterEstimate(const VCMFrameBuffer& frame,
1303                                            bool incomplete_frame) {
1304   if (frame.LatestPacketTimeMs() == -1) {
1305     return;
1306   }
1307   // No retransmitted frames should be a part of the jitter
1308   // estimate.
1309   UpdateJitterEstimate(frame.LatestPacketTimeMs(), frame.TimeStamp(),
1310                        frame.Length(), incomplete_frame);
1311 }
1312 
1313 // Must be called under the critical section |crit_sect_|. Should never be
1314 // called with retransmitted frames, they must be filtered out before this
1315 // function is called.
UpdateJitterEstimate(int64_t latest_packet_time_ms,uint32_t timestamp,unsigned int frame_size,bool incomplete_frame)1316 void VCMJitterBuffer::UpdateJitterEstimate(int64_t latest_packet_time_ms,
1317                                            uint32_t timestamp,
1318                                            unsigned int frame_size,
1319                                            bool incomplete_frame) {
1320   if (latest_packet_time_ms == -1) {
1321     return;
1322   }
1323   int64_t frame_delay;
1324   bool not_reordered = inter_frame_delay_.CalculateDelay(
1325       timestamp, &frame_delay, latest_packet_time_ms);
1326   // Filter out frames which have been reordered in time by the network
1327   if (not_reordered) {
1328     // Update the jitter estimate with the new samples
1329     jitter_estimate_.UpdateEstimate(frame_delay, frame_size, incomplete_frame);
1330   }
1331 }
1332 
WaitForRetransmissions()1333 bool VCMJitterBuffer::WaitForRetransmissions() {
1334   if (nack_mode_ == kNoNack) {
1335     // NACK disabled -> don't wait for retransmissions.
1336     return false;
1337   }
1338   // Evaluate if the RTT is higher than |high_rtt_nack_threshold_ms_|, and in
1339   // that case we don't wait for retransmissions.
1340   if (high_rtt_nack_threshold_ms_ >= 0 &&
1341       rtt_ms_ >= high_rtt_nack_threshold_ms_) {
1342     return false;
1343   }
1344   return true;
1345 }
1346 }  // namespace webrtc
1347