1 /* 2 * Copyright 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 11 #define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 12 13 #include <stddef.h> 14 #include <stdint.h> 15 16 #include <atomic> 17 #include <map> 18 #include <memory> 19 #include <vector> 20 21 #include "absl/types/optional.h" 22 #include "api/fec_controller.h" 23 #include "api/rtc_event_log/rtc_event_log.h" 24 #include "api/video/encoded_image.h" 25 #include "api/video/video_bitrate_allocation.h" 26 #include "api/video/video_bitrate_allocator.h" 27 #include "api/video/video_stream_encoder_interface.h" 28 #include "api/video_codecs/video_encoder.h" 29 #include "api/video_codecs/video_encoder_config.h" 30 #include "call/bitrate_allocator.h" 31 #include "call/rtp_config.h" 32 #include "call/rtp_transport_controller_send_interface.h" 33 #include "call/rtp_video_sender_interface.h" 34 #include "modules/include/module_common_types.h" 35 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 36 #include "modules/utility/include/process_thread.h" 37 #include "modules/video_coding/include/video_codec_interface.h" 38 #include "rtc_base/experiments/field_trial_parser.h" 39 #include "rtc_base/synchronization/mutex.h" 40 #include "rtc_base/task_queue.h" 41 #include "rtc_base/task_utils/repeating_task.h" 42 #include "rtc_base/thread_annotations.h" 43 #include "rtc_base/weak_ptr.h" 44 #include "video/encoder_rtcp_feedback.h" 45 #include "video/send_delay_stats.h" 46 #include "video/send_statistics_proxy.h" 47 #include "video/video_send_stream.h" 48 49 namespace webrtc { 50 namespace internal { 51 52 // Pacing buffer config; overridden by ALR config if provided. 53 struct PacingConfig { 54 PacingConfig(); 55 PacingConfig(const PacingConfig&); 56 PacingConfig& operator=(const PacingConfig&) = default; 57 ~PacingConfig(); 58 FieldTrialParameter<double> pacing_factor; 59 FieldTrialParameter<TimeDelta> max_pacing_delay; 60 }; 61 62 // VideoSendStreamImpl implements internal::VideoSendStream. 63 // It is created and destroyed on |worker_queue|. The intent is to decrease the 64 // need for locking and to ensure methods are called in sequence. 65 // Public methods except |DeliverRtcp| must be called on |worker_queue|. 66 // DeliverRtcp is called on the libjingle worker thread or a network thread. 67 // An encoder may deliver frames through the EncodedImageCallback on an 68 // arbitrary thread. 69 class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, 70 public VideoStreamEncoderInterface::EncoderSink, 71 public VideoBitrateAllocationObserver { 72 public: 73 VideoSendStreamImpl( 74 Clock* clock, 75 SendStatisticsProxy* stats_proxy, 76 rtc::TaskQueue* worker_queue, 77 RtcpRttStats* call_stats, 78 RtpTransportControllerSendInterface* transport, 79 BitrateAllocatorInterface* bitrate_allocator, 80 SendDelayStats* send_delay_stats, 81 VideoStreamEncoderInterface* video_stream_encoder, 82 RtcEventLog* event_log, 83 const VideoSendStream::Config* config, 84 int initial_encoder_max_bitrate, 85 double initial_encoder_bitrate_priority, 86 std::map<uint32_t, RtpState> suspended_ssrcs, 87 std::map<uint32_t, RtpPayloadState> suspended_payload_states, 88 VideoEncoderConfig::ContentType content_type, 89 std::unique_ptr<FecController> fec_controller); 90 ~VideoSendStreamImpl() override; 91 92 // RegisterProcessThread register |module_process_thread| with those objects 93 // that use it. Registration has to happen on the thread were 94 // |module_process_thread| was created (libjingle's worker thread). 95 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, 96 // maybe |worker_queue|. 97 void RegisterProcessThread(ProcessThread* module_process_thread); 98 void DeRegisterProcessThread(); 99 100 void DeliverRtcp(const uint8_t* packet, size_t length); 101 void UpdateActiveSimulcastLayers(const std::vector<bool> active_layers); 102 void Start(); 103 void Stop(); 104 105 // TODO(holmer): Move these to RtpTransportControllerSend. 106 std::map<uint32_t, RtpState> GetRtpStates() const; 107 108 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const; 109 110 absl::optional<float> configured_pacing_factor_; 111 112 private: 113 // Implements BitrateAllocatorObserver. 114 uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; 115 116 void OnEncoderConfigurationChanged( 117 std::vector<VideoStream> streams, 118 bool is_svc, 119 VideoEncoderConfig::ContentType content_type, 120 int min_transmit_bitrate_bps) override; 121 122 // Implements EncodedImageCallback. The implementation routes encoded frames 123 // to the |payload_router_| and |config.pre_encode_callback| if set. 124 // Called on an arbitrary encoder callback thread. 125 EncodedImageCallback::Result OnEncodedImage( 126 const EncodedImage& encoded_image, 127 const CodecSpecificInfo* codec_specific_info, 128 const RTPFragmentationHeader* fragmentation) override; 129 130 // Implements EncodedImageCallback. 131 void OnDroppedFrame(EncodedImageCallback::DropReason reason) override; 132 133 // Implements VideoBitrateAllocationObserver. 134 void OnBitrateAllocationUpdated( 135 const VideoBitrateAllocation& allocation) override; 136 137 // Starts monitoring and sends a keyframe. 138 void StartupVideoSendStream(); 139 // Removes the bitrate observer, stops monitoring and notifies the video 140 // encoder of the bitrate update. 141 void StopVideoSendStream() RTC_RUN_ON(worker_queue_); 142 143 void ConfigureProtection(); 144 void ConfigureSsrcs(); 145 void SignalEncoderTimedOut(); 146 void SignalEncoderActive(); 147 MediaStreamAllocationConfig GetAllocationConfig() const 148 RTC_RUN_ON(worker_queue_); 149 Clock* const clock_; 150 const bool has_alr_probing_; 151 const PacingConfig pacing_config_; 152 153 SendStatisticsProxy* const stats_proxy_; 154 const VideoSendStream::Config* const config_; 155 156 rtc::TaskQueue* const worker_queue_; 157 158 RepeatingTaskHandle check_encoder_activity_task_ 159 RTC_GUARDED_BY(worker_queue_); 160 161 std::atomic_bool activity_; 162 bool timed_out_ RTC_GUARDED_BY(worker_queue_); 163 164 RtpTransportControllerSendInterface* const transport_; 165 BitrateAllocatorInterface* const bitrate_allocator_; 166 167 Mutex ivf_writers_mutex_; 168 169 bool disable_padding_; 170 int max_padding_bitrate_; 171 int encoder_min_bitrate_bps_; 172 uint32_t encoder_max_bitrate_bps_; 173 uint32_t encoder_target_rate_bps_; 174 double encoder_bitrate_priority_; 175 bool has_packet_feedback_; 176 177 VideoStreamEncoderInterface* const video_stream_encoder_; 178 EncoderRtcpFeedback encoder_feedback_; 179 180 RtcpBandwidthObserver* const bandwidth_observer_; 181 RtpVideoSenderInterface* const rtp_video_sender_; 182 183 // |weak_ptr_| to our self. This is used since we can not call 184 // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy 185 // an existing WeakPtr. 186 rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_; 187 // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are 188 // invalidated before any other members are destroyed. 189 rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_; 190 191 // Context for the most recent and last sent video bitrate allocation. Used to 192 // throttle sending of similar bitrate allocations. 193 struct VbaSendContext { 194 VideoBitrateAllocation last_sent_allocation; 195 absl::optional<VideoBitrateAllocation> throttled_allocation; 196 int64_t last_send_time_ms; 197 }; 198 absl::optional<VbaSendContext> video_bitrate_allocation_context_ 199 RTC_GUARDED_BY(worker_queue_); 200 }; 201 } // namespace internal 202 } // namespace webrtc 203 #endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_ 204