1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ 12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ 13 14 #include <map> 15 #include <vector> 16 17 #include "webrtc/base/scoped_ptr.h" 18 #include "webrtc/base/scoped_ref_ptr.h" 19 #include "webrtc/base/thread_annotations.h" 20 #include "webrtc/call/bitrate_allocator.h" 21 #include "webrtc/common_types.h" 22 #include "webrtc/frame_callback.h" 23 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" 24 #include "webrtc/modules/video_coding/include/video_coding_defines.h" 25 #include "webrtc/modules/video_processing/include/video_processing.h" 26 #include "webrtc/typedefs.h" 27 #include "webrtc/video/video_capture_input.h" 28 29 namespace webrtc { 30 31 class BitrateAllocator; 32 class BitrateObserver; 33 class Config; 34 class CriticalSectionWrapper; 35 class EncodedImageCallback; 36 class PacedSender; 37 class PayloadRouter; 38 class ProcessThread; 39 class QMVideoSettingsCallback; 40 class SendStatisticsProxy; 41 class ViEBitrateObserver; 42 class ViEEffectFilter; 43 class VideoCodingModule; 44 45 class ViEEncoder : public RtcpIntraFrameObserver, 46 public VideoEncoderRateObserver, 47 public VCMPacketizationCallback, 48 public VCMSendStatisticsCallback, 49 public VideoCaptureCallback { 50 public: 51 friend class ViEBitrateObserver; 52 53 ViEEncoder(uint32_t number_of_cores, 54 ProcessThread* module_process_thread, 55 SendStatisticsProxy* stats_proxy, 56 I420FrameCallback* pre_encode_callback, 57 PacedSender* pacer, 58 BitrateAllocator* bitrate_allocator); 59 ~ViEEncoder(); 60 61 bool Init(); 62 63 // This function is assumed to be called before any frames are delivered and 64 // only once. 65 // Ideally this would be done in Init, but the dependencies between ViEEncoder 66 // and ViEChannel makes it really hard to do in a good way. 67 void StartThreadsAndSetSharedMembers( 68 rtc::scoped_refptr<PayloadRouter> send_payload_router, 69 VCMProtectionCallback* vcm_protection_callback); 70 71 // This function must be called before the corresponding ViEChannel is 72 // deleted. 73 void StopThreadsAndRemoveSharedMembers(); 74 75 void SetNetworkTransmissionState(bool is_transmitting); 76 77 // Returns the id of the owning channel. 78 int Owner() const; 79 80 // Drops incoming packets before they get to the encoder. 81 void Pause(); 82 void Restart(); 83 84 // Codec settings. 85 int32_t RegisterExternalEncoder(VideoEncoder* encoder, 86 uint8_t pl_type, 87 bool internal_source); 88 int32_t DeRegisterExternalEncoder(uint8_t pl_type); 89 int32_t SetEncoder(const VideoCodec& video_codec); 90 91 // Implementing VideoCaptureCallback. 92 void DeliverFrame(VideoFrame video_frame) override; 93 94 int32_t SendKeyFrame(); 95 96 uint32_t LastObservedBitrateBps() const; 97 int CodecTargetBitrate(uint32_t* bitrate) const; 98 // Loss protection. Must be called before SetEncoder() to have max packet size 99 // updated according to protection. 100 // TODO(pbos): Set protection method on construction or extract vcm_ outside 101 // this class and set it on construction there. 102 void SetProtectionMethod(bool nack, bool fec); 103 104 // Buffering mode. 105 void SetSenderBufferingMode(int target_delay_ms); 106 107 // Implements VideoEncoderRateObserver. 108 void OnSetRates(uint32_t bitrate_bps, int framerate) override; 109 110 // Implements VCMPacketizationCallback. 111 int32_t SendData(uint8_t payload_type, 112 const EncodedImage& encoded_image, 113 const RTPFragmentationHeader& fragmentation_header, 114 const RTPVideoHeader* rtp_video_hdr) override; 115 void OnEncoderImplementationName(const char* implementation_name) override; 116 117 // Implements VideoSendStatisticsCallback. 118 int32_t SendStatistics(const uint32_t bit_rate, 119 const uint32_t frame_rate) override; 120 121 // Implements RtcpIntraFrameObserver. 122 void OnReceivedIntraFrameRequest(uint32_t ssrc) override; 123 void OnReceivedSLI(uint32_t ssrc, uint8_t picture_id) override; 124 void OnReceivedRPSI(uint32_t ssrc, uint64_t picture_id) override; 125 void OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) override; 126 127 // Sets SSRCs for all streams. 128 void SetSsrcs(const std::vector<uint32_t>& ssrcs); 129 130 void SetMinTransmitBitrate(int min_transmit_bitrate_kbps); 131 132 // Lets the sender suspend video when the rate drops below 133 // |threshold_bps|, and turns back on when the rate goes back up above 134 // |threshold_bps| + |window_bps|. 135 void SuspendBelowMinBitrate(); 136 137 // New-style callbacks, used by VideoSendStream. 138 void RegisterPostEncodeImageCallback( 139 EncodedImageCallback* post_encode_callback); 140 141 int GetPaddingNeededBps() const; 142 143 protected: 144 // Called by BitrateObserver. 145 void OnNetworkChanged(uint32_t bitrate_bps, 146 uint8_t fraction_lost, 147 int64_t round_trip_time_ms); 148 149 private: 150 bool EncoderPaused() const EXCLUSIVE_LOCKS_REQUIRED(data_cs_); 151 void TraceFrameDropStart() EXCLUSIVE_LOCKS_REQUIRED(data_cs_); 152 void TraceFrameDropEnd() EXCLUSIVE_LOCKS_REQUIRED(data_cs_); 153 154 const uint32_t number_of_cores_; 155 156 const rtc::scoped_ptr<VideoProcessing> vp_; 157 const rtc::scoped_ptr<QMVideoSettingsCallback> qm_callback_; 158 const rtc::scoped_ptr<VideoCodingModule> vcm_; 159 rtc::scoped_refptr<PayloadRouter> send_payload_router_; 160 161 rtc::scoped_ptr<CriticalSectionWrapper> data_cs_; 162 rtc::scoped_ptr<BitrateObserver> bitrate_observer_; 163 164 SendStatisticsProxy* const stats_proxy_; 165 I420FrameCallback* const pre_encode_callback_; 166 PacedSender* const pacer_; 167 BitrateAllocator* const bitrate_allocator_; 168 169 // The time we last received an input frame or encoded frame. This is used to 170 // track when video is stopped long enough that we also want to stop sending 171 // padding. 172 int64_t time_of_last_frame_activity_ms_ GUARDED_BY(data_cs_); 173 VideoCodec encoder_config_ GUARDED_BY(data_cs_); 174 int min_transmit_bitrate_kbps_ GUARDED_BY(data_cs_); 175 uint32_t last_observed_bitrate_bps_ GUARDED_BY(data_cs_); 176 int target_delay_ms_ GUARDED_BY(data_cs_); 177 bool network_is_transmitting_ GUARDED_BY(data_cs_); 178 bool encoder_paused_ GUARDED_BY(data_cs_); 179 bool encoder_paused_and_dropped_frame_ GUARDED_BY(data_cs_); 180 std::map<unsigned int, int64_t> time_last_intra_request_ms_ 181 GUARDED_BY(data_cs_); 182 183 ProcessThread* module_process_thread_; 184 185 bool has_received_sli_ GUARDED_BY(data_cs_); 186 uint8_t picture_id_sli_ GUARDED_BY(data_cs_); 187 bool has_received_rpsi_ GUARDED_BY(data_cs_); 188 uint64_t picture_id_rpsi_ GUARDED_BY(data_cs_); 189 std::map<uint32_t, int> ssrc_streams_ GUARDED_BY(data_cs_); 190 191 bool video_suspended_ GUARDED_BY(data_cs_); 192 }; 193 194 } // namespace webrtc 195 196 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ 197