1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/video/vie_receiver.h"
12 
13 #include <vector>
14 
15 #include "webrtc/base/logging.h"
16 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
17 #include "webrtc/modules/rtp_rtcp/include/fec_receiver.h"
18 #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
19 #include "webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
20 #include "webrtc/modules/rtp_rtcp/include/rtp_cvo.h"
21 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
22 #include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
23 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
24 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
25 #include "webrtc/modules/video_coding/include/video_coding.h"
26 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
27 #include "webrtc/system_wrappers/include/metrics.h"
28 #include "webrtc/system_wrappers/include/tick_util.h"
29 #include "webrtc/system_wrappers/include/timestamp_extrapolator.h"
30 #include "webrtc/system_wrappers/include/trace.h"
31 
32 namespace webrtc {
33 
34 static const int kPacketLogIntervalMs = 10000;
35 
ViEReceiver(VideoCodingModule * module_vcm,RemoteBitrateEstimator * remote_bitrate_estimator,RtpFeedback * rtp_feedback)36 ViEReceiver::ViEReceiver(VideoCodingModule* module_vcm,
37                          RemoteBitrateEstimator* remote_bitrate_estimator,
38                          RtpFeedback* rtp_feedback)
39     : receive_cs_(CriticalSectionWrapper::CreateCriticalSection()),
40       clock_(Clock::GetRealTimeClock()),
41       rtp_header_parser_(RtpHeaderParser::Create()),
42       rtp_payload_registry_(
43           new RTPPayloadRegistry(RTPPayloadStrategy::CreateStrategy(false))),
44       rtp_receiver_(
45           RtpReceiver::CreateVideoReceiver(clock_,
46                                            this,
47                                            rtp_feedback,
48                                            rtp_payload_registry_.get())),
49       rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
50       fec_receiver_(FecReceiver::Create(this)),
51       rtp_rtcp_(NULL),
52       vcm_(module_vcm),
53       remote_bitrate_estimator_(remote_bitrate_estimator),
54       ntp_estimator_(new RemoteNtpTimeEstimator(clock_)),
55       receiving_(false),
56       restored_packet_in_use_(false),
57       receiving_ast_enabled_(false),
58       receiving_cvo_enabled_(false),
59       receiving_tsn_enabled_(false),
60       last_packet_log_ms_(-1) {
61   assert(remote_bitrate_estimator);
62 }
63 
~ViEReceiver()64 ViEReceiver::~ViEReceiver() {
65   UpdateHistograms();
66 }
67 
UpdateHistograms()68 void ViEReceiver::UpdateHistograms() {
69   FecPacketCounter counter = fec_receiver_->GetPacketCounter();
70   if (counter.num_packets > 0) {
71     RTC_HISTOGRAM_PERCENTAGE_SPARSE(
72         "WebRTC.Video.ReceivedFecPacketsInPercent",
73         static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets));
74   }
75   if (counter.num_fec_packets > 0) {
76     RTC_HISTOGRAM_PERCENTAGE_SPARSE(
77         "WebRTC.Video.RecoveredMediaPacketsInPercentOfFec",
78         static_cast<int>(counter.num_recovered_packets * 100 /
79                          counter.num_fec_packets));
80   }
81 }
82 
SetReceiveCodec(const VideoCodec & video_codec)83 bool ViEReceiver::SetReceiveCodec(const VideoCodec& video_codec) {
84   int8_t old_pltype = -1;
85   if (rtp_payload_registry_->ReceivePayloadType(video_codec.plName,
86                                                 kVideoPayloadTypeFrequency,
87                                                 0,
88                                                 video_codec.maxBitrate,
89                                                 &old_pltype) != -1) {
90     rtp_payload_registry_->DeRegisterReceivePayload(old_pltype);
91   }
92 
93   return RegisterPayload(video_codec);
94 }
95 
RegisterPayload(const VideoCodec & video_codec)96 bool ViEReceiver::RegisterPayload(const VideoCodec& video_codec) {
97   return rtp_receiver_->RegisterReceivePayload(video_codec.plName,
98                                                video_codec.plType,
99                                                kVideoPayloadTypeFrequency,
100                                                0,
101                                                video_codec.maxBitrate) == 0;
102 }
103 
SetNackStatus(bool enable,int max_nack_reordering_threshold)104 void ViEReceiver::SetNackStatus(bool enable,
105                                 int max_nack_reordering_threshold) {
106   if (!enable) {
107     // Reset the threshold back to the lower default threshold when NACK is
108     // disabled since we no longer will be receiving retransmissions.
109     max_nack_reordering_threshold = kDefaultMaxReorderingThreshold;
110   }
111   rtp_receive_statistics_->SetMaxReorderingThreshold(
112       max_nack_reordering_threshold);
113   rtp_receiver_->SetNACKStatus(enable ? kNackRtcp : kNackOff);
114 }
115 
SetRtxPayloadType(int payload_type,int associated_payload_type)116 void ViEReceiver::SetRtxPayloadType(int payload_type,
117                                     int associated_payload_type) {
118   rtp_payload_registry_->SetRtxPayloadType(payload_type,
119                                            associated_payload_type);
120 }
121 
SetUseRtxPayloadMappingOnRestore(bool val)122 void ViEReceiver::SetUseRtxPayloadMappingOnRestore(bool val) {
123   rtp_payload_registry_->set_use_rtx_payload_mapping_on_restore(val);
124 }
125 
SetRtxSsrc(uint32_t ssrc)126 void ViEReceiver::SetRtxSsrc(uint32_t ssrc) {
127   rtp_payload_registry_->SetRtxSsrc(ssrc);
128 }
129 
GetRtxSsrc(uint32_t * ssrc) const130 bool ViEReceiver::GetRtxSsrc(uint32_t* ssrc) const {
131   return rtp_payload_registry_->GetRtxSsrc(ssrc);
132 }
133 
IsFecEnabled() const134 bool ViEReceiver::IsFecEnabled() const {
135   return rtp_payload_registry_->ulpfec_payload_type() > -1;
136 }
137 
GetRemoteSsrc() const138 uint32_t ViEReceiver::GetRemoteSsrc() const {
139   return rtp_receiver_->SSRC();
140 }
141 
GetCsrcs(uint32_t * csrcs) const142 int ViEReceiver::GetCsrcs(uint32_t* csrcs) const {
143   return rtp_receiver_->CSRCs(csrcs);
144 }
145 
SetRtpRtcpModule(RtpRtcp * module)146 void ViEReceiver::SetRtpRtcpModule(RtpRtcp* module) {
147   rtp_rtcp_ = module;
148 }
149 
GetRtpReceiver() const150 RtpReceiver* ViEReceiver::GetRtpReceiver() const {
151   return rtp_receiver_.get();
152 }
153 
RegisterRtpRtcpModules(const std::vector<RtpRtcp * > & rtp_modules)154 void ViEReceiver::RegisterRtpRtcpModules(
155     const std::vector<RtpRtcp*>& rtp_modules) {
156   CriticalSectionScoped cs(receive_cs_.get());
157   // Only change the "simulcast" modules, the base module can be accessed
158   // without a lock whereas the simulcast modules require locking as they can be
159   // changed in runtime.
160   rtp_rtcp_simulcast_ =
161       std::vector<RtpRtcp*>(rtp_modules.begin() + 1, rtp_modules.end());
162 }
163 
SetReceiveTimestampOffsetStatus(bool enable,int id)164 bool ViEReceiver::SetReceiveTimestampOffsetStatus(bool enable, int id) {
165   if (enable) {
166     return rtp_header_parser_->RegisterRtpHeaderExtension(
167         kRtpExtensionTransmissionTimeOffset, id);
168   } else {
169     return rtp_header_parser_->DeregisterRtpHeaderExtension(
170         kRtpExtensionTransmissionTimeOffset);
171   }
172 }
173 
SetReceiveAbsoluteSendTimeStatus(bool enable,int id)174 bool ViEReceiver::SetReceiveAbsoluteSendTimeStatus(bool enable, int id) {
175   if (enable) {
176     if (rtp_header_parser_->RegisterRtpHeaderExtension(
177         kRtpExtensionAbsoluteSendTime, id)) {
178       receiving_ast_enabled_ = true;
179       return true;
180     } else {
181       return false;
182     }
183   } else {
184     receiving_ast_enabled_ = false;
185     return rtp_header_parser_->DeregisterRtpHeaderExtension(
186         kRtpExtensionAbsoluteSendTime);
187   }
188 }
189 
SetReceiveVideoRotationStatus(bool enable,int id)190 bool ViEReceiver::SetReceiveVideoRotationStatus(bool enable, int id) {
191   if (enable) {
192     if (rtp_header_parser_->RegisterRtpHeaderExtension(
193             kRtpExtensionVideoRotation, id)) {
194       receiving_cvo_enabled_ = true;
195       return true;
196     } else {
197       return false;
198     }
199   } else {
200     receiving_cvo_enabled_ = false;
201     return rtp_header_parser_->DeregisterRtpHeaderExtension(
202         kRtpExtensionVideoRotation);
203   }
204 }
205 
SetReceiveTransportSequenceNumber(bool enable,int id)206 bool ViEReceiver::SetReceiveTransportSequenceNumber(bool enable, int id) {
207   if (enable) {
208     if (rtp_header_parser_->RegisterRtpHeaderExtension(
209             kRtpExtensionTransportSequenceNumber, id)) {
210       receiving_tsn_enabled_ = true;
211       return true;
212     } else {
213       return false;
214     }
215   } else {
216     receiving_tsn_enabled_ = false;
217     return rtp_header_parser_->DeregisterRtpHeaderExtension(
218         kRtpExtensionTransportSequenceNumber);
219   }
220 }
221 
ReceivedRTPPacket(const void * rtp_packet,size_t rtp_packet_length,const PacketTime & packet_time)222 int ViEReceiver::ReceivedRTPPacket(const void* rtp_packet,
223                                    size_t rtp_packet_length,
224                                    const PacketTime& packet_time) {
225   return InsertRTPPacket(static_cast<const uint8_t*>(rtp_packet),
226                          rtp_packet_length, packet_time);
227 }
228 
ReceivedRTCPPacket(const void * rtcp_packet,size_t rtcp_packet_length)229 int ViEReceiver::ReceivedRTCPPacket(const void* rtcp_packet,
230                                     size_t rtcp_packet_length) {
231   return InsertRTCPPacket(static_cast<const uint8_t*>(rtcp_packet),
232                           rtcp_packet_length);
233 }
234 
OnReceivedPayloadData(const uint8_t * payload_data,const size_t payload_size,const WebRtcRTPHeader * rtp_header)235 int32_t ViEReceiver::OnReceivedPayloadData(const uint8_t* payload_data,
236                                            const size_t payload_size,
237                                            const WebRtcRTPHeader* rtp_header) {
238   WebRtcRTPHeader rtp_header_with_ntp = *rtp_header;
239   rtp_header_with_ntp.ntp_time_ms =
240       ntp_estimator_->Estimate(rtp_header->header.timestamp);
241   if (vcm_->IncomingPacket(payload_data,
242                            payload_size,
243                            rtp_header_with_ntp) != 0) {
244     // Check this...
245     return -1;
246   }
247   return 0;
248 }
249 
OnRecoveredPacket(const uint8_t * rtp_packet,size_t rtp_packet_length)250 bool ViEReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
251                                     size_t rtp_packet_length) {
252   RTPHeader header;
253   if (!rtp_header_parser_->Parse(rtp_packet, rtp_packet_length, &header)) {
254     return false;
255   }
256   header.payload_type_frequency = kVideoPayloadTypeFrequency;
257   bool in_order = IsPacketInOrder(header);
258   return ReceivePacket(rtp_packet, rtp_packet_length, header, in_order);
259 }
260 
InsertRTPPacket(const uint8_t * rtp_packet,size_t rtp_packet_length,const PacketTime & packet_time)261 int ViEReceiver::InsertRTPPacket(const uint8_t* rtp_packet,
262                                  size_t rtp_packet_length,
263                                  const PacketTime& packet_time) {
264   {
265     CriticalSectionScoped cs(receive_cs_.get());
266     if (!receiving_) {
267       return -1;
268     }
269   }
270 
271   RTPHeader header;
272   if (!rtp_header_parser_->Parse(rtp_packet, rtp_packet_length,
273                                  &header)) {
274     return -1;
275   }
276   size_t payload_length = rtp_packet_length - header.headerLength;
277   int64_t arrival_time_ms;
278   int64_t now_ms = clock_->TimeInMilliseconds();
279   if (packet_time.timestamp != -1)
280     arrival_time_ms = (packet_time.timestamp + 500) / 1000;
281   else
282     arrival_time_ms = now_ms;
283 
284   {
285     // Periodically log the RTP header of incoming packets.
286     CriticalSectionScoped cs(receive_cs_.get());
287     if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) {
288       std::stringstream ss;
289       ss << "Packet received on SSRC: " << header.ssrc << " with payload type: "
290          << static_cast<int>(header.payloadType) << ", timestamp: "
291          << header.timestamp << ", sequence number: " << header.sequenceNumber
292          << ", arrival time: " << arrival_time_ms;
293       if (header.extension.hasTransmissionTimeOffset)
294         ss << ", toffset: " << header.extension.transmissionTimeOffset;
295       if (header.extension.hasAbsoluteSendTime)
296         ss << ", abs send time: " << header.extension.absoluteSendTime;
297       LOG(LS_INFO) << ss.str();
298       last_packet_log_ms_ = now_ms;
299     }
300   }
301 
302   remote_bitrate_estimator_->IncomingPacket(arrival_time_ms, payload_length,
303                                             header, true);
304   header.payload_type_frequency = kVideoPayloadTypeFrequency;
305 
306   bool in_order = IsPacketInOrder(header);
307   rtp_payload_registry_->SetIncomingPayloadType(header);
308   int ret = ReceivePacket(rtp_packet, rtp_packet_length, header, in_order)
309       ? 0
310       : -1;
311   // Update receive statistics after ReceivePacket.
312   // Receive statistics will be reset if the payload type changes (make sure
313   // that the first packet is included in the stats).
314   rtp_receive_statistics_->IncomingPacket(
315       header, rtp_packet_length, IsPacketRetransmitted(header, in_order));
316   return ret;
317 }
318 
ReceivePacket(const uint8_t * packet,size_t packet_length,const RTPHeader & header,bool in_order)319 bool ViEReceiver::ReceivePacket(const uint8_t* packet,
320                                 size_t packet_length,
321                                 const RTPHeader& header,
322                                 bool in_order) {
323   if (rtp_payload_registry_->IsEncapsulated(header)) {
324     return ParseAndHandleEncapsulatingHeader(packet, packet_length, header);
325   }
326   const uint8_t* payload = packet + header.headerLength;
327   assert(packet_length >= header.headerLength);
328   size_t payload_length = packet_length - header.headerLength;
329   PayloadUnion payload_specific;
330   if (!rtp_payload_registry_->GetPayloadSpecifics(header.payloadType,
331                                                   &payload_specific)) {
332     return false;
333   }
334   return rtp_receiver_->IncomingRtpPacket(header, payload, payload_length,
335                                           payload_specific, in_order);
336 }
337 
ParseAndHandleEncapsulatingHeader(const uint8_t * packet,size_t packet_length,const RTPHeader & header)338 bool ViEReceiver::ParseAndHandleEncapsulatingHeader(const uint8_t* packet,
339                                                     size_t packet_length,
340                                                     const RTPHeader& header) {
341   if (rtp_payload_registry_->IsRed(header)) {
342     int8_t ulpfec_pt = rtp_payload_registry_->ulpfec_payload_type();
343     if (packet[header.headerLength] == ulpfec_pt) {
344       rtp_receive_statistics_->FecPacketReceived(header, packet_length);
345       // Notify vcm about received FEC packets to avoid NACKing these packets.
346       NotifyReceiverOfFecPacket(header);
347     }
348     if (fec_receiver_->AddReceivedRedPacket(
349             header, packet, packet_length, ulpfec_pt) != 0) {
350       return false;
351     }
352     return fec_receiver_->ProcessReceivedFec() == 0;
353   } else if (rtp_payload_registry_->IsRtx(header)) {
354     if (header.headerLength + header.paddingLength == packet_length) {
355       // This is an empty packet and should be silently dropped before trying to
356       // parse the RTX header.
357       return true;
358     }
359     // Remove the RTX header and parse the original RTP header.
360     if (packet_length < header.headerLength)
361       return false;
362     if (packet_length > sizeof(restored_packet_))
363       return false;
364     CriticalSectionScoped cs(receive_cs_.get());
365     if (restored_packet_in_use_) {
366       LOG(LS_WARNING) << "Multiple RTX headers detected, dropping packet.";
367       return false;
368     }
369     if (!rtp_payload_registry_->RestoreOriginalPacket(
370             restored_packet_, packet, &packet_length, rtp_receiver_->SSRC(),
371             header)) {
372       LOG(LS_WARNING) << "Incoming RTX packet: Invalid RTP header";
373       return false;
374     }
375     restored_packet_in_use_ = true;
376     bool ret = OnRecoveredPacket(restored_packet_, packet_length);
377     restored_packet_in_use_ = false;
378     return ret;
379   }
380   return false;
381 }
382 
NotifyReceiverOfFecPacket(const RTPHeader & header)383 void ViEReceiver::NotifyReceiverOfFecPacket(const RTPHeader& header) {
384   int8_t last_media_payload_type =
385       rtp_payload_registry_->last_received_media_payload_type();
386   if (last_media_payload_type < 0) {
387     LOG(LS_WARNING) << "Failed to get last media payload type.";
388     return;
389   }
390   // Fake an empty media packet.
391   WebRtcRTPHeader rtp_header = {};
392   rtp_header.header = header;
393   rtp_header.header.payloadType = last_media_payload_type;
394   rtp_header.header.paddingLength = 0;
395   PayloadUnion payload_specific;
396   if (!rtp_payload_registry_->GetPayloadSpecifics(last_media_payload_type,
397                                                   &payload_specific)) {
398     LOG(LS_WARNING) << "Failed to get payload specifics.";
399     return;
400   }
401   rtp_header.type.Video.codec = payload_specific.Video.videoCodecType;
402   rtp_header.type.Video.rotation = kVideoRotation_0;
403   if (header.extension.hasVideoRotation) {
404     rtp_header.type.Video.rotation =
405         ConvertCVOByteToVideoRotation(header.extension.videoRotation);
406   }
407   OnReceivedPayloadData(NULL, 0, &rtp_header);
408 }
409 
InsertRTCPPacket(const uint8_t * rtcp_packet,size_t rtcp_packet_length)410 int ViEReceiver::InsertRTCPPacket(const uint8_t* rtcp_packet,
411                                   size_t rtcp_packet_length) {
412   {
413     CriticalSectionScoped cs(receive_cs_.get());
414     if (!receiving_) {
415       return -1;
416     }
417 
418     for (RtpRtcp* rtp_rtcp : rtp_rtcp_simulcast_)
419       rtp_rtcp->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
420   }
421   assert(rtp_rtcp_);  // Should be set by owner at construction time.
422   int ret = rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
423   if (ret != 0) {
424     return ret;
425   }
426 
427   int64_t rtt = 0;
428   rtp_rtcp_->RTT(rtp_receiver_->SSRC(), &rtt, NULL, NULL, NULL);
429   if (rtt == 0) {
430     // Waiting for valid rtt.
431     return 0;
432   }
433   uint32_t ntp_secs = 0;
434   uint32_t ntp_frac = 0;
435   uint32_t rtp_timestamp = 0;
436   if (0 != rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, NULL, NULL,
437                                 &rtp_timestamp)) {
438     // Waiting for RTCP.
439     return 0;
440   }
441   ntp_estimator_->UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
442 
443   return 0;
444 }
445 
StartReceive()446 void ViEReceiver::StartReceive() {
447   CriticalSectionScoped cs(receive_cs_.get());
448   receiving_ = true;
449 }
450 
StopReceive()451 void ViEReceiver::StopReceive() {
452   CriticalSectionScoped cs(receive_cs_.get());
453   receiving_ = false;
454 }
455 
GetReceiveStatistics() const456 ReceiveStatistics* ViEReceiver::GetReceiveStatistics() const {
457   return rtp_receive_statistics_.get();
458 }
459 
IsPacketInOrder(const RTPHeader & header) const460 bool ViEReceiver::IsPacketInOrder(const RTPHeader& header) const {
461   StreamStatistician* statistician =
462       rtp_receive_statistics_->GetStatistician(header.ssrc);
463   if (!statistician)
464     return false;
465   return statistician->IsPacketInOrder(header.sequenceNumber);
466 }
467 
IsPacketRetransmitted(const RTPHeader & header,bool in_order) const468 bool ViEReceiver::IsPacketRetransmitted(const RTPHeader& header,
469                                         bool in_order) const {
470   // Retransmissions are handled separately if RTX is enabled.
471   if (rtp_payload_registry_->RtxEnabled())
472     return false;
473   StreamStatistician* statistician =
474       rtp_receive_statistics_->GetStatistician(header.ssrc);
475   if (!statistician)
476     return false;
477   // Check if this is a retransmission.
478   int64_t min_rtt = 0;
479   rtp_rtcp_->RTT(rtp_receiver_->SSRC(), NULL, NULL, &min_rtt, NULL);
480   return !in_order &&
481       statistician->IsRetransmitOfOldPacket(header, min_rtt);
482 }
483 }  // namespace webrtc
484