1 /*
2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/base/checks.h"
12 #include "webrtc/base/logging.h"
13 #include "webrtc/base/trace_event.h"
14 #include "webrtc/common_types.h"
15 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
17 #include "webrtc/modules/video_coding/encoded_frame.h"
18 #include "webrtc/modules/video_coding/jitter_buffer.h"
19 #include "webrtc/modules/video_coding/packet.h"
20 #include "webrtc/modules/video_coding/video_coding_impl.h"
21 #include "webrtc/system_wrappers/include/clock.h"
22 
23 // #define DEBUG_DECODER_BIT_STREAM
24 
25 namespace webrtc {
26 namespace vcm {
27 
VideoReceiver(Clock * clock,EventFactory * event_factory)28 VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory)
29     : clock_(clock),
30       process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
31       _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()),
32       _timing(clock_),
33       _receiver(&_timing, clock_, event_factory),
34       _decodedFrameCallback(&_timing, clock_),
35       _frameTypeCallback(NULL),
36       _receiveStatsCallback(NULL),
37       _decoderTimingCallback(NULL),
38       _packetRequestCallback(NULL),
39       render_buffer_callback_(NULL),
40       _decoder(NULL),
41 #ifdef DEBUG_DECODER_BIT_STREAM
42       _bitStreamBeforeDecoder(NULL),
43 #endif
44       _frameFromFile(),
45       _scheduleKeyRequest(false),
46       max_nack_list_size_(0),
47       pre_decode_image_callback_(NULL),
48       _codecDataBase(nullptr, nullptr),
49       _receiveStatsTimer(1000, clock_),
50       _retransmissionTimer(10, clock_),
51       _keyRequestTimer(500, clock_) {
52   assert(clock_);
53 #ifdef DEBUG_DECODER_BIT_STREAM
54   _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb");
55 #endif
56 }
57 
~VideoReceiver()58 VideoReceiver::~VideoReceiver() {
59   delete _receiveCritSect;
60 #ifdef DEBUG_DECODER_BIT_STREAM
61   fclose(_bitStreamBeforeDecoder);
62 #endif
63 }
64 
Process()65 int32_t VideoReceiver::Process() {
66   int32_t returnValue = VCM_OK;
67 
68   // Receive-side statistics
69   if (_receiveStatsTimer.TimeUntilProcess() == 0) {
70     _receiveStatsTimer.Processed();
71     CriticalSectionScoped cs(process_crit_sect_.get());
72     if (_receiveStatsCallback != NULL) {
73       uint32_t bitRate;
74       uint32_t frameRate;
75       _receiver.ReceiveStatistics(&bitRate, &frameRate);
76       _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate);
77     }
78 
79     if (_decoderTimingCallback != NULL) {
80       int decode_ms;
81       int max_decode_ms;
82       int current_delay_ms;
83       int target_delay_ms;
84       int jitter_buffer_ms;
85       int min_playout_delay_ms;
86       int render_delay_ms;
87       _timing.GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
88                          &target_delay_ms, &jitter_buffer_ms,
89                          &min_playout_delay_ms, &render_delay_ms);
90       _decoderTimingCallback->OnDecoderTiming(
91           decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
92           jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
93     }
94 
95     // Size of render buffer.
96     if (render_buffer_callback_) {
97       int buffer_size_ms = _receiver.RenderBufferSizeMs();
98       render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms);
99     }
100   }
101 
102   // Key frame requests
103   if (_keyRequestTimer.TimeUntilProcess() == 0) {
104     _keyRequestTimer.Processed();
105     bool request_key_frame = false;
106     {
107       CriticalSectionScoped cs(process_crit_sect_.get());
108       request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL;
109     }
110     if (request_key_frame) {
111       const int32_t ret = RequestKeyFrame();
112       if (ret != VCM_OK && returnValue == VCM_OK) {
113         returnValue = ret;
114       }
115     }
116   }
117 
118   // Packet retransmission requests
119   // TODO(holmer): Add API for changing Process interval and make sure it's
120   // disabled when NACK is off.
121   if (_retransmissionTimer.TimeUntilProcess() == 0) {
122     _retransmissionTimer.Processed();
123     bool callback_registered = false;
124     uint16_t length;
125     {
126       CriticalSectionScoped cs(process_crit_sect_.get());
127       length = max_nack_list_size_;
128       callback_registered = _packetRequestCallback != NULL;
129     }
130     if (callback_registered && length > 0) {
131       // Collect sequence numbers from the default receiver.
132       bool request_key_frame = false;
133       std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
134       int32_t ret = VCM_OK;
135       if (request_key_frame) {
136         ret = RequestKeyFrame();
137         if (ret != VCM_OK && returnValue == VCM_OK) {
138           returnValue = ret;
139         }
140       }
141       if (ret == VCM_OK && !nackList.empty()) {
142         CriticalSectionScoped cs(process_crit_sect_.get());
143         if (_packetRequestCallback != NULL) {
144           _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
145         }
146       }
147     }
148   }
149 
150   return returnValue;
151 }
152 
TimeUntilNextProcess()153 int64_t VideoReceiver::TimeUntilNextProcess() {
154   int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
155   if (_receiver.NackMode() != kNoNack) {
156     // We need a Process call more often if we are relying on
157     // retransmissions
158     timeUntilNextProcess =
159         VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
160   }
161   timeUntilNextProcess =
162       VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
163 
164   return timeUntilNextProcess;
165 }
166 
SetReceiveChannelParameters(int64_t rtt)167 int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
168   CriticalSectionScoped receiveCs(_receiveCritSect);
169   _receiver.UpdateRtt(rtt);
170   return 0;
171 }
172 
173 // Enable or disable a video protection method.
174 // Note: This API should be deprecated, as it does not offer a distinction
175 // between the protection method and decoding with or without errors. If such a
176 // behavior is desired, use the following API: SetReceiverRobustnessMode.
SetVideoProtection(VCMVideoProtection videoProtection,bool enable)177 int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
178                                           bool enable) {
179   // By default, do not decode with errors.
180   _receiver.SetDecodeErrorMode(kNoErrors);
181   switch (videoProtection) {
182     case kProtectionNack: {
183       RTC_DCHECK(enable);
184       _receiver.SetNackMode(kNack, -1, -1);
185       break;
186     }
187 
188     case kProtectionNackFEC: {
189       CriticalSectionScoped cs(_receiveCritSect);
190       RTC_DCHECK(enable);
191       _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
192       _receiver.SetDecodeErrorMode(kNoErrors);
193       break;
194     }
195     case kProtectionFEC:
196     case kProtectionNone:
197       // No receiver-side protection.
198       RTC_DCHECK(enable);
199       _receiver.SetNackMode(kNoNack, -1, -1);
200       _receiver.SetDecodeErrorMode(kWithErrors);
201       break;
202   }
203   return VCM_OK;
204 }
205 
206 // Register a receive callback. Will be called whenever there is a new frame
207 // ready for rendering.
RegisterReceiveCallback(VCMReceiveCallback * receiveCallback)208 int32_t VideoReceiver::RegisterReceiveCallback(
209     VCMReceiveCallback* receiveCallback) {
210   CriticalSectionScoped cs(_receiveCritSect);
211   _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
212   return VCM_OK;
213 }
214 
RegisterReceiveStatisticsCallback(VCMReceiveStatisticsCallback * receiveStats)215 int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
216     VCMReceiveStatisticsCallback* receiveStats) {
217   CriticalSectionScoped cs(process_crit_sect_.get());
218   _receiver.RegisterStatsCallback(receiveStats);
219   _receiveStatsCallback = receiveStats;
220   return VCM_OK;
221 }
222 
RegisterDecoderTimingCallback(VCMDecoderTimingCallback * decoderTiming)223 int32_t VideoReceiver::RegisterDecoderTimingCallback(
224     VCMDecoderTimingCallback* decoderTiming) {
225   CriticalSectionScoped cs(process_crit_sect_.get());
226   _decoderTimingCallback = decoderTiming;
227   return VCM_OK;
228 }
229 
230 // Register an externally defined decoder object.
RegisterExternalDecoder(VideoDecoder * externalDecoder,uint8_t payloadType)231 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
232                                             uint8_t payloadType) {
233   CriticalSectionScoped cs(_receiveCritSect);
234   if (externalDecoder == NULL) {
235     // Make sure the VCM updates the decoder next time it decodes.
236     _decoder = NULL;
237     RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
238     return;
239   }
240   _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
241 }
242 
243 // Register a frame type request callback.
RegisterFrameTypeCallback(VCMFrameTypeCallback * frameTypeCallback)244 int32_t VideoReceiver::RegisterFrameTypeCallback(
245     VCMFrameTypeCallback* frameTypeCallback) {
246   CriticalSectionScoped cs(process_crit_sect_.get());
247   _frameTypeCallback = frameTypeCallback;
248   return VCM_OK;
249 }
250 
RegisterPacketRequestCallback(VCMPacketRequestCallback * callback)251 int32_t VideoReceiver::RegisterPacketRequestCallback(
252     VCMPacketRequestCallback* callback) {
253   CriticalSectionScoped cs(process_crit_sect_.get());
254   _packetRequestCallback = callback;
255   return VCM_OK;
256 }
257 
RegisterRenderBufferSizeCallback(VCMRenderBufferSizeCallback * callback)258 int VideoReceiver::RegisterRenderBufferSizeCallback(
259     VCMRenderBufferSizeCallback* callback) {
260   CriticalSectionScoped cs(process_crit_sect_.get());
261   render_buffer_callback_ = callback;
262   return VCM_OK;
263 }
264 
TriggerDecoderShutdown()265 void VideoReceiver::TriggerDecoderShutdown() {
266   _receiver.TriggerDecoderShutdown();
267 }
268 
269 // Decode next frame, blocking.
270 // Should be called as often as possible to get the most out of the decoder.
Decode(uint16_t maxWaitTimeMs)271 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
272   int64_t nextRenderTimeMs;
273   bool prefer_late_decoding = false;
274   {
275     CriticalSectionScoped cs(_receiveCritSect);
276     prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
277   }
278 
279   VCMEncodedFrame* frame = _receiver.FrameForDecoding(
280       maxWaitTimeMs, &nextRenderTimeMs, prefer_late_decoding);
281 
282   if (!frame)
283     return VCM_FRAME_NOT_READY;
284 
285   CriticalSectionScoped cs(_receiveCritSect);
286 
287   // If this frame was too late, we should adjust the delay accordingly
288   _timing.UpdateCurrentDelay(frame->RenderTimeMs(),
289                              clock_->TimeInMilliseconds());
290 
291   if (pre_decode_image_callback_) {
292     EncodedImage encoded_image(frame->EncodedImage());
293     int qp = -1;
294     if (qp_parser_.GetQp(*frame, &qp)) {
295       encoded_image.qp_ = qp;
296     }
297     pre_decode_image_callback_->Encoded(encoded_image, frame->CodecSpecific(),
298                                         NULL);
299   }
300 
301 #ifdef DEBUG_DECODER_BIT_STREAM
302   if (_bitStreamBeforeDecoder != NULL) {
303     // Write bit stream to file for debugging purposes
304     if (fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) !=
305         frame->Length()) {
306       return -1;
307     }
308   }
309 #endif
310   const int32_t ret = Decode(*frame);
311   _receiver.ReleaseFrame(frame);
312   return ret;
313 }
314 
RequestSliceLossIndication(const uint64_t pictureID) const315 int32_t VideoReceiver::RequestSliceLossIndication(
316     const uint64_t pictureID) const {
317   TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
318   CriticalSectionScoped cs(process_crit_sect_.get());
319   if (_frameTypeCallback != NULL) {
320     const int32_t ret =
321         _frameTypeCallback->SliceLossIndicationRequest(pictureID);
322     if (ret < 0) {
323       return ret;
324     }
325   } else {
326     return VCM_MISSING_CALLBACK;
327   }
328   return VCM_OK;
329 }
330 
RequestKeyFrame()331 int32_t VideoReceiver::RequestKeyFrame() {
332   TRACE_EVENT0("webrtc", "RequestKeyFrame");
333   CriticalSectionScoped process_cs(process_crit_sect_.get());
334   if (_frameTypeCallback != NULL) {
335     const int32_t ret = _frameTypeCallback->RequestKeyFrame();
336     if (ret < 0) {
337       return ret;
338     }
339     _scheduleKeyRequest = false;
340   } else {
341     return VCM_MISSING_CALLBACK;
342   }
343   return VCM_OK;
344 }
345 
346 // Must be called from inside the receive side critical section.
Decode(const VCMEncodedFrame & frame)347 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
348   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode",
349                           "type", frame.FrameType());
350   // Change decoder if payload type has changed
351   _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
352   if (_decoder == NULL) {
353     return VCM_NO_CODEC_REGISTERED;
354   }
355   // Decode a frame
356   int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());
357 
358   // Check for failed decoding, run frame type request callback if needed.
359   bool request_key_frame = false;
360   if (ret < 0) {
361     if (ret == VCM_ERROR_REQUEST_SLI) {
362       return RequestSliceLossIndication(
363           _decodedFrameCallback.LastReceivedPictureID() + 1);
364     } else {
365       request_key_frame = true;
366     }
367   } else if (ret == VCM_REQUEST_SLI) {
368     ret = RequestSliceLossIndication(
369         _decodedFrameCallback.LastReceivedPictureID() + 1);
370   }
371   if (!frame.Complete() || frame.MissingFrame()) {
372     request_key_frame = true;
373     ret = VCM_OK;
374   }
375   if (request_key_frame) {
376     CriticalSectionScoped cs(process_crit_sect_.get());
377     _scheduleKeyRequest = true;
378   }
379   TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
380   return ret;
381 }
382 
383 // Reset the decoder state
ResetDecoder()384 int32_t VideoReceiver::ResetDecoder() {
385   bool reset_key_request = false;
386   {
387     CriticalSectionScoped cs(_receiveCritSect);
388     if (_decoder != NULL) {
389       _receiver.Reset();
390       _timing.Reset();
391       reset_key_request = true;
392       _decoder->Reset();
393     }
394   }
395   if (reset_key_request) {
396     CriticalSectionScoped cs(process_crit_sect_.get());
397     _scheduleKeyRequest = false;
398   }
399   return VCM_OK;
400 }
401 
402 // Register possible receive codecs, can be called multiple times
RegisterReceiveCodec(const VideoCodec * receiveCodec,int32_t numberOfCores,bool requireKeyFrame)403 int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
404                                             int32_t numberOfCores,
405                                             bool requireKeyFrame) {
406   CriticalSectionScoped cs(_receiveCritSect);
407   if (receiveCodec == NULL) {
408     return VCM_PARAMETER_ERROR;
409   }
410   if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
411                                            requireKeyFrame)) {
412     return -1;
413   }
414   return 0;
415 }
416 
417 // Get current received codec
ReceiveCodec(VideoCodec * currentReceiveCodec) const418 int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
419   CriticalSectionScoped cs(_receiveCritSect);
420   if (currentReceiveCodec == NULL) {
421     return VCM_PARAMETER_ERROR;
422   }
423   return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
424 }
425 
426 // Get current received codec
ReceiveCodec() const427 VideoCodecType VideoReceiver::ReceiveCodec() const {
428   CriticalSectionScoped cs(_receiveCritSect);
429   return _codecDataBase.ReceiveCodec();
430 }
431 
432 // Incoming packet from network parsed and ready for decode, non blocking.
IncomingPacket(const uint8_t * incomingPayload,size_t payloadLength,const WebRtcRTPHeader & rtpInfo)433 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
434                                       size_t payloadLength,
435                                       const WebRtcRTPHeader& rtpInfo) {
436   if (rtpInfo.frameType == kVideoFrameKey) {
437     TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
438                  rtpInfo.header.sequenceNumber);
439   }
440   if (incomingPayload == NULL) {
441     // The jitter buffer doesn't handle non-zero payload lengths for packets
442     // without payload.
443     // TODO(holmer): We should fix this in the jitter buffer.
444     payloadLength = 0;
445   }
446   const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
447   int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width,
448                                        rtpInfo.type.Video.height);
449   // TODO(holmer): Investigate if this somehow should use the key frame
450   // request scheduling to throttle the requests.
451   if (ret == VCM_FLUSH_INDICATOR) {
452     RequestKeyFrame();
453     ResetDecoder();
454   } else if (ret < 0) {
455     return ret;
456   }
457   return VCM_OK;
458 }
459 
460 // Minimum playout delay (used for lip-sync). This is the minimum delay required
461 // to sync with audio. Not included in  VideoCodingModule::Delay()
462 // Defaults to 0 ms.
SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs)463 int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
464   _timing.set_min_playout_delay(minPlayoutDelayMs);
465   return VCM_OK;
466 }
467 
468 // The estimated delay caused by rendering, defaults to
469 // kDefaultRenderDelayMs = 10 ms
SetRenderDelay(uint32_t timeMS)470 int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
471   _timing.set_render_delay(timeMS);
472   return VCM_OK;
473 }
474 
475 // Current video delay
Delay() const476 int32_t VideoReceiver::Delay() const {
477   return _timing.TargetVideoDelay();
478 }
479 
DiscardedPackets() const480 uint32_t VideoReceiver::DiscardedPackets() const {
481   return _receiver.DiscardedPackets();
482 }
483 
SetReceiverRobustnessMode(ReceiverRobustness robustnessMode,VCMDecodeErrorMode decode_error_mode)484 int VideoReceiver::SetReceiverRobustnessMode(
485     ReceiverRobustness robustnessMode,
486     VCMDecodeErrorMode decode_error_mode) {
487   CriticalSectionScoped cs(_receiveCritSect);
488   switch (robustnessMode) {
489     case VideoCodingModule::kNone:
490       _receiver.SetNackMode(kNoNack, -1, -1);
491       break;
492     case VideoCodingModule::kHardNack:
493       // Always wait for retransmissions (except when decoding with errors).
494       _receiver.SetNackMode(kNack, -1, -1);
495       break;
496     case VideoCodingModule::kSoftNack:
497 #if 1
498       assert(false);  // TODO(hlundin): Not completed.
499       return VCM_NOT_IMPLEMENTED;
500 #else
501       // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
502       // extra delay when RTT is above kLowRttNackMs.
503       _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
504       break;
505 #endif
506     case VideoCodingModule::kReferenceSelection:
507 #if 1
508       assert(false);  // TODO(hlundin): Not completed.
509       return VCM_NOT_IMPLEMENTED;
510 #else
511       if (decode_error_mode == kNoErrors) {
512         return VCM_PARAMETER_ERROR;
513       }
514       _receiver.SetNackMode(kNoNack, -1, -1);
515       break;
516 #endif
517   }
518   _receiver.SetDecodeErrorMode(decode_error_mode);
519   return VCM_OK;
520 }
521 
SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode)522 void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
523   CriticalSectionScoped cs(_receiveCritSect);
524   _receiver.SetDecodeErrorMode(decode_error_mode);
525 }
526 
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)527 void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
528                                     int max_packet_age_to_nack,
529                                     int max_incomplete_time_ms) {
530   if (max_nack_list_size != 0) {
531     CriticalSectionScoped process_cs(process_crit_sect_.get());
532     max_nack_list_size_ = max_nack_list_size;
533   }
534   _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
535                             max_incomplete_time_ms);
536 }
537 
SetMinReceiverDelay(int desired_delay_ms)538 int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
539   return _receiver.SetMinReceiverDelay(desired_delay_ms);
540 }
541 
RegisterPreDecodeImageCallback(EncodedImageCallback * observer)542 void VideoReceiver::RegisterPreDecodeImageCallback(
543     EncodedImageCallback* observer) {
544   CriticalSectionScoped cs(_receiveCritSect);
545   pre_decode_image_callback_ = observer;
546 }
547 
548 }  // namespace vcm
549 }  // namespace webrtc
550