1 /*
2  *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
12 #define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
13 
14 #include <atomic>
15 #include <map>
16 #include <memory>
17 #include <string>
18 #include <unordered_map>
19 #include <utility>
20 #include <vector>
21 
22 #include "absl/types/optional.h"
23 #include "api/adaptation/resource.h"
24 #include "api/rtp_parameters.h"
25 #include "api/scoped_refptr.h"
26 #include "api/task_queue/task_queue_base.h"
27 #include "api/video/video_adaptation_counters.h"
28 #include "api/video/video_adaptation_reason.h"
29 #include "api/video/video_frame.h"
30 #include "api/video/video_source_interface.h"
31 #include "api/video/video_stream_encoder_observer.h"
32 #include "api/video_codecs/video_codec.h"
33 #include "api/video_codecs/video_encoder.h"
34 #include "api/video_codecs/video_encoder_config.h"
35 #include "call/adaptation/resource_adaptation_processor_interface.h"
36 #include "call/adaptation/video_stream_adapter.h"
37 #include "call/adaptation/video_stream_input_state_provider.h"
38 #include "rtc_base/experiments/quality_scaler_settings.h"
39 #include "rtc_base/ref_count.h"
40 #include "rtc_base/strings/string_builder.h"
41 #include "rtc_base/synchronization/mutex.h"
42 #include "rtc_base/task_queue.h"
43 #include "system_wrappers/include/clock.h"
44 #include "video/adaptation/encode_usage_resource.h"
45 #include "video/adaptation/overuse_frame_detector.h"
46 #include "video/adaptation/quality_rampup_experiment_helper.h"
47 #include "video/adaptation/quality_scaler_resource.h"
48 #include "video/adaptation/video_stream_encoder_resource.h"
49 
50 namespace webrtc {
51 
52 // The assumed input frame size if we have not yet received a frame.
53 // TODO(hbos): This is 144p - why are we assuming super low quality? Seems like
54 // a bad heuristic.
55 extern const int kDefaultInputPixelsWidth;
56 extern const int kDefaultInputPixelsHeight;
57 
58 // Owns adaptation-related Resources pertaining to a single VideoStreamEncoder
59 // and passes on the relevant input from the encoder to the resources. The
60 // resources provide resource usage states to the ResourceAdaptationProcessor
61 // which is responsible for reconfiguring streams in order not to overuse
62 // resources.
63 //
64 // The manager is also involved with various mitigations not part of the
65 // ResourceAdaptationProcessor code such as the inital frame dropping.
66 class VideoStreamEncoderResourceManager
67     : public VideoSourceRestrictionsListener,
68       public ResourceLimitationsListener,
69       public QualityRampUpExperimentListener {
70  public:
71   VideoStreamEncoderResourceManager(
72       VideoStreamInputStateProvider* input_state_provider,
73       VideoStreamEncoderObserver* encoder_stats_observer,
74       Clock* clock,
75       bool experiment_cpu_load_estimator,
76       std::unique_ptr<OveruseFrameDetector> overuse_detector,
77       DegradationPreferenceProvider* degradation_preference_provider);
78   ~VideoStreamEncoderResourceManager() override;
79 
80   void Initialize(rtc::TaskQueue* encoder_queue,
81                   rtc::TaskQueue* resource_adaptation_queue);
82   void SetAdaptationProcessor(
83       ResourceAdaptationProcessorInterface* adaptation_processor,
84       VideoStreamAdapter* stream_adapter);
85 
86   // TODO(https://crbug.com/webrtc/11563): The degradation preference is a
87   // setting of the Processor, it does not belong to the Manager - can we get
88   // rid of this?
89   void SetDegradationPreferences(DegradationPreference degradation_preference);
90   DegradationPreference degradation_preference() const;
91 
92   // Starts the encode usage resource. The quality scaler resource is
93   // automatically started on being configured.
94   void StartEncodeUsageResource();
95   // Stops the encode usage and quality scaler resources if not already stopped.
96   void StopManagedResources();
97 
98   // Settings that affect the VideoStreamEncoder-specific resources.
99   void SetEncoderSettings(EncoderSettings encoder_settings);
100   void SetStartBitrate(DataRate start_bitrate);
101   void SetTargetBitrate(DataRate target_bitrate);
102   void SetEncoderRates(
103       const VideoEncoder::RateControlParameters& encoder_rates);
104   // TODO(https://crbug.com/webrtc/11338): This can be made private if we
105   // configure on SetDegredationPreference and SetEncoderSettings.
106   void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
107 
108   // Methods corresponding to different points in the encoding pipeline.
109   void OnFrameDroppedDueToSize();
110   void OnMaybeEncodeFrame();
111   void OnEncodeStarted(const VideoFrame& cropped_frame,
112                        int64_t time_when_first_seen_us);
113   void OnEncodeCompleted(const EncodedImage& encoded_image,
114                          int64_t time_sent_in_us,
115                          absl::optional<int> encode_duration_us);
116   void OnFrameDropped(EncodedImageCallback::DropReason reason);
117 
118   // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
119   // to update legacy getStats().
120   void MapResourceToReason(rtc::scoped_refptr<Resource> resource,
121                            VideoAdaptationReason reason);
122   std::vector<rtc::scoped_refptr<Resource>> MappedResources() const;
123   std::vector<AdaptationConstraint*> AdaptationConstraints() const;
124   std::vector<AdaptationListener*> AdaptationListeners() const;
125   rtc::scoped_refptr<QualityScalerResource>
126   quality_scaler_resource_for_testing();
127   // If true, the VideoStreamEncoder should eexecute its logic to maybe drop
128   // frames baseed on size and bitrate.
129   bool DropInitialFrames() const;
130 
131   // VideoSourceRestrictionsListener implementation.
132   // Updates |video_source_restrictions_|.
133   void OnVideoSourceRestrictionsUpdated(
134       VideoSourceRestrictions restrictions,
135       const VideoAdaptationCounters& adaptation_counters,
136       rtc::scoped_refptr<Resource> reason,
137       const VideoSourceRestrictions& unfiltered_restrictions) override;
138   void OnResourceLimitationChanged(
139       rtc::scoped_refptr<Resource> resource,
140       const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
141           resource_limitations) override;
142 
143   // QualityRampUpExperimentListener implementation.
144   void OnQualityRampUp() override;
145 
146  private:
147   class InitialFrameDropper;
148 
149   VideoAdaptationReason GetReasonFromResource(
150       rtc::scoped_refptr<Resource> resource) const;
151 
152   CpuOveruseOptions GetCpuOveruseOptions() const;
153   int LastInputFrameSizeOrDefault() const;
154 
155   // Calculates an up-to-date value of the target frame rate and informs the
156   // |encode_usage_resource_| of the new value.
157   void MaybeUpdateTargetFrameRate();
158 
159   // Use nullopt to disable quality scaling.
160   void UpdateQualityScalerSettings(
161       absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
162 
163   void UpdateStatsAdaptationSettings() const;
164 
165   static std::string ActiveCountsToString(
166       const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
167           active_counts);
168 
169   // TODO(hbos): Add tests for manager's constraints.
170   // Does not trigger adaptations, only prevents adapting up resolution.
171   class BitrateConstraint : public rtc::RefCountInterface,
172                             public AdaptationConstraint {
173    public:
174     explicit BitrateConstraint(VideoStreamEncoderResourceManager* manager);
175     ~BitrateConstraint() override = default;
176 
177     void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue);
178     void OnEncoderSettingsUpdated(
179         absl::optional<EncoderSettings> encoder_settings);
180     void OnEncoderTargetBitrateUpdated(
181         absl::optional<uint32_t> encoder_target_bitrate_bps);
182 
183     // AdaptationConstraint implementation.
Name()184     std::string Name() const override { return "BitrateConstraint"; }
185     bool IsAdaptationUpAllowed(
186         const VideoStreamInputState& input_state,
187         const VideoSourceRestrictions& restrictions_before,
188         const VideoSourceRestrictions& restrictions_after,
189         rtc::scoped_refptr<Resource> reason_resource) const override;
190 
191    private:
192     // The |manager_| must be alive as long as this resource is added to the
193     // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called.
194     VideoStreamEncoderResourceManager* const manager_;
195     TaskQueueBase* resource_adaptation_queue_;
196     absl::optional<EncoderSettings> encoder_settings_
197         RTC_GUARDED_BY(resource_adaptation_queue_);
198     absl::optional<uint32_t> encoder_target_bitrate_bps_
199         RTC_GUARDED_BY(resource_adaptation_queue_);
200   };
201 
202   // Does not trigger adaptations, only prevents adapting up in BALANCED.
203   class BalancedConstraint : public rtc::RefCountInterface,
204                              public AdaptationConstraint {
205    public:
206     BalancedConstraint(
207         VideoStreamEncoderResourceManager* manager,
208         DegradationPreferenceProvider* degradation_preference_provider);
209     ~BalancedConstraint() override = default;
210 
211     void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue);
212     void OnEncoderTargetBitrateUpdated(
213         absl::optional<uint32_t> encoder_target_bitrate_bps);
214 
215     // AdaptationConstraint implementation.
Name()216     std::string Name() const override { return "BalancedConstraint"; }
217     bool IsAdaptationUpAllowed(
218         const VideoStreamInputState& input_state,
219         const VideoSourceRestrictions& restrictions_before,
220         const VideoSourceRestrictions& restrictions_after,
221         rtc::scoped_refptr<Resource> reason_resource) const override;
222 
223    private:
224     // The |manager_| must be alive as long as this resource is added to the
225     // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called.
226     VideoStreamEncoderResourceManager* const manager_;
227     TaskQueueBase* resource_adaptation_queue_;
228     absl::optional<uint32_t> encoder_target_bitrate_bps_
229         RTC_GUARDED_BY(resource_adaptation_queue_);
230     DegradationPreferenceProvider* degradation_preference_provider_;
231   };
232 
233   DegradationPreferenceProvider* const degradation_preference_provider_;
234   const rtc::scoped_refptr<BitrateConstraint> bitrate_constraint_;
235   const rtc::scoped_refptr<BalancedConstraint> balanced_constraint_;
236   const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
237   const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
238 
239   rtc::TaskQueue* encoder_queue_;
240   rtc::TaskQueue* resource_adaptation_queue_;
241   VideoStreamInputStateProvider* const input_state_provider_
242       RTC_GUARDED_BY(encoder_queue_);
243   ResourceAdaptationProcessorInterface* adaptation_processor_
244       RTC_GUARDED_BY(resource_adaptation_queue_);
245   VideoStreamAdapter* stream_adapter_
246       RTC_GUARDED_BY(resource_adaptation_queue_);
247   // Thread-safe.
248   VideoStreamEncoderObserver* const encoder_stats_observer_;
249 
250   DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
251   VideoSourceRestrictions video_source_restrictions_
252       RTC_GUARDED_BY(encoder_queue_);
253 
254   const BalancedDegradationSettings balanced_settings_;
255   Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
256   const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);
257   const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_
258       RTC_GUARDED_BY(encoder_queue_);
259   const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_);
260   absl::optional<uint32_t> encoder_target_bitrate_bps_
261       RTC_GUARDED_BY(encoder_queue_);
262   absl::optional<VideoEncoder::RateControlParameters> encoder_rates_
263       RTC_GUARDED_BY(encoder_queue_);
264   std::unique_ptr<QualityRampUpExperimentHelper> quality_rampup_experiment_
265       RTC_GUARDED_BY(encoder_queue_);
266   absl::optional<EncoderSettings> encoder_settings_
267       RTC_GUARDED_BY(encoder_queue_);
268 
269   // Ties a resource to a reason for statistical reporting. This AdaptReason is
270   // also used by this module to make decisions about how to adapt up/down.
271   struct ResourceAndReason {
ResourceAndReasonResourceAndReason272     ResourceAndReason(rtc::scoped_refptr<Resource> resource,
273                       VideoAdaptationReason reason)
274         : resource(resource), reason(reason) {}
275     virtual ~ResourceAndReason() = default;
276 
277     const rtc::scoped_refptr<Resource> resource;
278     const VideoAdaptationReason reason;
279   };
280   mutable Mutex resource_lock_;
281   std::vector<ResourceAndReason> resources_ RTC_GUARDED_BY(&resource_lock_);
282 };
283 
284 }  // namespace webrtc
285 
286 #endif  // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
287