1 /*
2  *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "video/adaptation/video_stream_encoder_resource_manager.h"
12 
13 #include <cmath>
14 #include <limits>
15 #include <memory>
16 #include <utility>
17 
18 #include "absl/algorithm/container.h"
19 #include "absl/base/macros.h"
20 #include "api/adaptation/resource.h"
21 #include "api/task_queue/task_queue_base.h"
22 #include "api/video/video_adaptation_reason.h"
23 #include "api/video/video_source_interface.h"
24 #include "call/adaptation/video_source_restrictions.h"
25 #include "rtc_base/logging.h"
26 #include "rtc_base/numerics/safe_conversions.h"
27 #include "rtc_base/ref_counted_object.h"
28 #include "rtc_base/strings/string_builder.h"
29 #include "rtc_base/time_utils.h"
30 
31 namespace webrtc {
32 
33 const int kDefaultInputPixelsWidth = 176;
34 const int kDefaultInputPixelsHeight = 144;
35 
36 namespace {
37 
IsResolutionScalingEnabled(DegradationPreference degradation_preference)38 bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
39   return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
40          degradation_preference == DegradationPreference::BALANCED;
41 }
42 
IsFramerateScalingEnabled(DegradationPreference degradation_preference)43 bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
44   return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
45          degradation_preference == DegradationPreference::BALANCED;
46 }
47 
ToString(VideoAdaptationReason reason)48 std::string ToString(VideoAdaptationReason reason) {
49   switch (reason) {
50     case VideoAdaptationReason::kQuality:
51       return "quality";
52     case VideoAdaptationReason::kCpu:
53       return "cpu";
54   }
55 }
56 
57 }  // namespace
58 
59 class VideoStreamEncoderResourceManager::InitialFrameDropper {
60  public:
InitialFrameDropper(rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)61   explicit InitialFrameDropper(
62       rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)
63       : quality_scaler_resource_(quality_scaler_resource),
64         quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
65         has_seen_first_bwe_drop_(false),
66         set_start_bitrate_(DataRate::Zero()),
67         set_start_bitrate_time_ms_(0),
68         initial_framedrop_(0) {
69     RTC_DCHECK(quality_scaler_resource_);
70   }
71 
72   // Output signal.
DropInitialFrames() const73   bool DropInitialFrames() const {
74     return initial_framedrop_ < kMaxInitialFramedrop;
75   }
76 
77   // Input signals.
SetStartBitrate(DataRate start_bitrate,int64_t now_ms)78   void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
79     set_start_bitrate_ = start_bitrate;
80     set_start_bitrate_time_ms_ = now_ms;
81   }
82 
SetTargetBitrate(DataRate target_bitrate,int64_t now_ms)83   void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
84     if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
85         quality_scaler_resource_->is_started() &&
86         quality_scaler_settings_.InitialBitrateIntervalMs() &&
87         quality_scaler_settings_.InitialBitrateFactor()) {
88       int64_t diff_ms = now_ms - set_start_bitrate_time_ms_;
89       if (diff_ms <
90               quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
91           (target_bitrate <
92            (set_start_bitrate_ *
93             quality_scaler_settings_.InitialBitrateFactor().value()))) {
94         RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
95                          << set_start_bitrate_.bps()
96                          << ", target bitrate: " << target_bitrate.bps();
97         initial_framedrop_ = 0;
98         has_seen_first_bwe_drop_ = true;
99       }
100     }
101   }
102 
OnFrameDroppedDueToSize()103   void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
104 
OnMaybeEncodeFrame()105   void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
106 
OnQualityScalerSettingsUpdated()107   void OnQualityScalerSettingsUpdated() {
108     if (quality_scaler_resource_->is_started()) {
109       // Restart frame drops due to size.
110       initial_framedrop_ = 0;
111     } else {
112       // Quality scaling disabled so we shouldn't drop initial frames.
113       initial_framedrop_ = kMaxInitialFramedrop;
114     }
115   }
116 
117  private:
118   // The maximum number of frames to drop at beginning of stream to try and
119   // achieve desired bitrate.
120   static const int kMaxInitialFramedrop = 4;
121 
122   const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
123   const QualityScalerSettings quality_scaler_settings_;
124   bool has_seen_first_bwe_drop_;
125   DataRate set_start_bitrate_;
126   int64_t set_start_bitrate_time_ms_;
127   // Counts how many frames we've dropped in the initial framedrop phase.
128   int initial_framedrop_;
129 };
130 
BitrateConstraint(VideoStreamEncoderResourceManager * manager)131 VideoStreamEncoderResourceManager::BitrateConstraint::BitrateConstraint(
132     VideoStreamEncoderResourceManager* manager)
133     : manager_(manager),
134       resource_adaptation_queue_(nullptr),
135       encoder_settings_(absl::nullopt),
136       encoder_target_bitrate_bps_(absl::nullopt) {}
137 
SetAdaptationQueue(TaskQueueBase * resource_adaptation_queue)138 void VideoStreamEncoderResourceManager::BitrateConstraint::SetAdaptationQueue(
139     TaskQueueBase* resource_adaptation_queue) {
140   resource_adaptation_queue_ = resource_adaptation_queue;
141 }
142 
143 void VideoStreamEncoderResourceManager::BitrateConstraint::
OnEncoderSettingsUpdated(absl::optional<EncoderSettings> encoder_settings)144     OnEncoderSettingsUpdated(absl::optional<EncoderSettings> encoder_settings) {
145   RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
146   resource_adaptation_queue_->PostTask(
147       ToQueuedTask([this_ref = rtc::scoped_refptr<BitrateConstraint>(this),
148                     encoder_settings] {
149         RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
150         this_ref->encoder_settings_ = std::move(encoder_settings);
151       }));
152 }
153 
154 void VideoStreamEncoderResourceManager::BitrateConstraint::
OnEncoderTargetBitrateUpdated(absl::optional<uint32_t> encoder_target_bitrate_bps)155     OnEncoderTargetBitrateUpdated(
156         absl::optional<uint32_t> encoder_target_bitrate_bps) {
157   RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
158   resource_adaptation_queue_->PostTask(
159       ToQueuedTask([this_ref = rtc::scoped_refptr<BitrateConstraint>(this),
160                     encoder_target_bitrate_bps] {
161         RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
162         this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps;
163       }));
164 }
165 
166 bool VideoStreamEncoderResourceManager::BitrateConstraint::
IsAdaptationUpAllowed(const VideoStreamInputState & input_state,const VideoSourceRestrictions & restrictions_before,const VideoSourceRestrictions & restrictions_after,rtc::scoped_refptr<Resource> reason_resource) const167     IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
168                           const VideoSourceRestrictions& restrictions_before,
169                           const VideoSourceRestrictions& restrictions_after,
170                           rtc::scoped_refptr<Resource> reason_resource) const {
171   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
172   VideoAdaptationReason reason =
173       manager_->GetReasonFromResource(reason_resource);
174   // If increasing resolution due to kQuality, make sure bitrate limits are not
175   // violated.
176   // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating bitrate
177   // constraints if adapting due to CPU? Shouldn't this condition be checked
178   // regardless of reason?
179   if (reason == VideoAdaptationReason::kQuality &&
180       DidIncreaseResolution(restrictions_before, restrictions_after)) {
181     uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0);
182     absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
183         encoder_settings_.has_value()
184             ? encoder_settings_->encoder_info()
185                   .GetEncoderBitrateLimitsForResolution(
186                       // Need some sort of expected resulting pixels to be used
187                       // instead of unrestricted.
188                       GetHigherResolutionThan(
189                           input_state.frame_size_pixels().value()))
190             : absl::nullopt;
191     if (bitrate_limits.has_value() && bitrate_bps != 0) {
192       RTC_DCHECK_GE(bitrate_limits->frame_size_pixels,
193                     input_state.frame_size_pixels().value());
194       return bitrate_bps >=
195              static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
196     }
197   }
198   return true;
199 }
200 
BalancedConstraint(VideoStreamEncoderResourceManager * manager,DegradationPreferenceProvider * degradation_preference_provider)201 VideoStreamEncoderResourceManager::BalancedConstraint::BalancedConstraint(
202     VideoStreamEncoderResourceManager* manager,
203     DegradationPreferenceProvider* degradation_preference_provider)
204     : manager_(manager),
205       resource_adaptation_queue_(nullptr),
206       encoder_target_bitrate_bps_(absl::nullopt),
207       degradation_preference_provider_(degradation_preference_provider) {
208   RTC_DCHECK(manager_);
209   RTC_DCHECK(degradation_preference_provider_);
210 }
211 
SetAdaptationQueue(TaskQueueBase * resource_adaptation_queue)212 void VideoStreamEncoderResourceManager::BalancedConstraint::SetAdaptationQueue(
213     TaskQueueBase* resource_adaptation_queue) {
214   resource_adaptation_queue_ = resource_adaptation_queue;
215 }
216 
217 void VideoStreamEncoderResourceManager::BalancedConstraint::
OnEncoderTargetBitrateUpdated(absl::optional<uint32_t> encoder_target_bitrate_bps)218     OnEncoderTargetBitrateUpdated(
219         absl::optional<uint32_t> encoder_target_bitrate_bps) {
220   RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
221   resource_adaptation_queue_->PostTask(
222       ToQueuedTask([this_ref = rtc::scoped_refptr<BalancedConstraint>(this),
223                     encoder_target_bitrate_bps] {
224         RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
225         this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps;
226       }));
227 }
228 
229 bool VideoStreamEncoderResourceManager::BalancedConstraint::
IsAdaptationUpAllowed(const VideoStreamInputState & input_state,const VideoSourceRestrictions & restrictions_before,const VideoSourceRestrictions & restrictions_after,rtc::scoped_refptr<Resource> reason_resource) const230     IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
231                           const VideoSourceRestrictions& restrictions_before,
232                           const VideoSourceRestrictions& restrictions_after,
233                           rtc::scoped_refptr<Resource> reason_resource) const {
234   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
235   VideoAdaptationReason reason =
236       manager_->GetReasonFromResource(reason_resource);
237   // Don't adapt if BalancedDegradationSettings applies and determines this will
238   // exceed bitrate constraints.
239   // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating
240   // balanced settings if adapting due CPU? Shouldn't this condition be checked
241   // regardless of reason?
242   if (reason == VideoAdaptationReason::kQuality &&
243       degradation_preference_provider_->degradation_preference() ==
244           DegradationPreference::BALANCED &&
245       !manager_->balanced_settings_.CanAdaptUp(
246           input_state.video_codec_type(),
247           input_state.frame_size_pixels().value(),
248           encoder_target_bitrate_bps_.value_or(0))) {
249     return false;
250   }
251   if (reason == VideoAdaptationReason::kQuality &&
252       DidIncreaseResolution(restrictions_before, restrictions_after) &&
253       !manager_->balanced_settings_.CanAdaptUpResolution(
254           input_state.video_codec_type(),
255           input_state.frame_size_pixels().value(),
256           encoder_target_bitrate_bps_.value_or(0))) {
257     return false;
258   }
259   return true;
260 }
261 
VideoStreamEncoderResourceManager(VideoStreamInputStateProvider * input_state_provider,VideoStreamEncoderObserver * encoder_stats_observer,Clock * clock,bool experiment_cpu_load_estimator,std::unique_ptr<OveruseFrameDetector> overuse_detector,DegradationPreferenceProvider * degradation_preference_provider)262 VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
263     VideoStreamInputStateProvider* input_state_provider,
264     VideoStreamEncoderObserver* encoder_stats_observer,
265     Clock* clock,
266     bool experiment_cpu_load_estimator,
267     std::unique_ptr<OveruseFrameDetector> overuse_detector,
268     DegradationPreferenceProvider* degradation_preference_provider)
269     : degradation_preference_provider_(degradation_preference_provider),
270       bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
271       balanced_constraint_(new rtc::RefCountedObject<BalancedConstraint>(
272           this,
273           degradation_preference_provider_)),
274       encode_usage_resource_(
275           EncodeUsageResource::Create(std::move(overuse_detector))),
276       quality_scaler_resource_(
277           QualityScalerResource::Create(degradation_preference_provider_)),
278       encoder_queue_(nullptr),
279       resource_adaptation_queue_(nullptr),
280       input_state_provider_(input_state_provider),
281       adaptation_processor_(nullptr),
282       encoder_stats_observer_(encoder_stats_observer),
283       degradation_preference_(DegradationPreference::DISABLED),
284       video_source_restrictions_(),
285       clock_(clock),
286       experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
287       initial_frame_dropper_(
288           std::make_unique<InitialFrameDropper>(quality_scaler_resource_)),
289       quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
290       encoder_target_bitrate_bps_(absl::nullopt),
291       quality_rampup_experiment_(
292           QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)),
293       encoder_settings_(absl::nullopt) {
294   RTC_CHECK(degradation_preference_provider_);
295   RTC_CHECK(encoder_stats_observer_);
296   MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu);
297   MapResourceToReason(quality_scaler_resource_,
298                       VideoAdaptationReason::kQuality);
299 }
300 
~VideoStreamEncoderResourceManager()301 VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {}
302 
Initialize(rtc::TaskQueue * encoder_queue,rtc::TaskQueue * resource_adaptation_queue)303 void VideoStreamEncoderResourceManager::Initialize(
304     rtc::TaskQueue* encoder_queue,
305     rtc::TaskQueue* resource_adaptation_queue) {
306   RTC_DCHECK(!encoder_queue_);
307   RTC_DCHECK(encoder_queue);
308   RTC_DCHECK(!resource_adaptation_queue_);
309   RTC_DCHECK(resource_adaptation_queue);
310   encoder_queue_ = encoder_queue;
311   resource_adaptation_queue_ = resource_adaptation_queue;
312   bitrate_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
313   balanced_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
314   encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
315   encode_usage_resource_->RegisterAdaptationTaskQueue(
316       resource_adaptation_queue_->Get());
317   quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
318   quality_scaler_resource_->RegisterAdaptationTaskQueue(
319       resource_adaptation_queue_->Get());
320 }
321 
SetAdaptationProcessor(ResourceAdaptationProcessorInterface * adaptation_processor,VideoStreamAdapter * stream_adapter)322 void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
323     ResourceAdaptationProcessorInterface* adaptation_processor,
324     VideoStreamAdapter* stream_adapter) {
325   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
326   adaptation_processor_ = adaptation_processor;
327   stream_adapter_ = stream_adapter;
328 }
329 
SetDegradationPreferences(DegradationPreference degradation_preference)330 void VideoStreamEncoderResourceManager::SetDegradationPreferences(
331     DegradationPreference degradation_preference) {
332   RTC_DCHECK_RUN_ON(encoder_queue_);
333   degradation_preference_ = degradation_preference;
334   UpdateStatsAdaptationSettings();
335 }
336 
337 DegradationPreference
degradation_preference() const338 VideoStreamEncoderResourceManager::degradation_preference() const {
339   RTC_DCHECK_RUN_ON(encoder_queue_);
340   return degradation_preference_;
341 }
342 
StartEncodeUsageResource()343 void VideoStreamEncoderResourceManager::StartEncodeUsageResource() {
344   RTC_DCHECK_RUN_ON(encoder_queue_);
345   RTC_DCHECK(!encode_usage_resource_->is_started());
346   RTC_DCHECK(encoder_settings_.has_value());
347   encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
348 }
349 
StopManagedResources()350 void VideoStreamEncoderResourceManager::StopManagedResources() {
351   RTC_DCHECK_RUN_ON(encoder_queue_);
352   encode_usage_resource_->StopCheckForOveruse();
353   quality_scaler_resource_->StopCheckForOveruse();
354 }
355 
MapResourceToReason(rtc::scoped_refptr<Resource> resource,VideoAdaptationReason reason)356 void VideoStreamEncoderResourceManager::MapResourceToReason(
357     rtc::scoped_refptr<Resource> resource,
358     VideoAdaptationReason reason) {
359   MutexLock lock(&resource_lock_);
360   RTC_DCHECK(resource);
361   RTC_DCHECK(absl::c_find_if(resources_,
362                              [resource](const ResourceAndReason& r) {
363                                return r.resource == resource;
364                              }) == resources_.end())
365       << "Resource " << resource->Name() << " already was inserted";
366   resources_.emplace_back(resource, reason);
367 }
368 
369 std::vector<rtc::scoped_refptr<Resource>>
MappedResources() const370 VideoStreamEncoderResourceManager::MappedResources() const {
371   MutexLock lock(&resource_lock_);
372   std::vector<rtc::scoped_refptr<Resource>> resources;
373   for (auto const& resource_and_reason : resources_) {
374     resources.push_back(resource_and_reason.resource);
375   }
376   return resources;
377 }
378 
379 std::vector<AdaptationConstraint*>
AdaptationConstraints() const380 VideoStreamEncoderResourceManager::AdaptationConstraints() const {
381   return {bitrate_constraint_, balanced_constraint_};
382 }
383 
384 std::vector<AdaptationListener*>
AdaptationListeners() const385 VideoStreamEncoderResourceManager::AdaptationListeners() const {
386   return {quality_scaler_resource_};
387 }
388 
389 rtc::scoped_refptr<QualityScalerResource>
quality_scaler_resource_for_testing()390 VideoStreamEncoderResourceManager::quality_scaler_resource_for_testing() {
391   MutexLock lock(&resource_lock_);
392   return quality_scaler_resource_;
393 }
394 
SetEncoderSettings(EncoderSettings encoder_settings)395 void VideoStreamEncoderResourceManager::SetEncoderSettings(
396     EncoderSettings encoder_settings) {
397   RTC_DCHECK_RUN_ON(encoder_queue_);
398   encoder_settings_ = std::move(encoder_settings);
399   bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
400   MaybeUpdateTargetFrameRate();
401 }
402 
SetStartBitrate(DataRate start_bitrate)403 void VideoStreamEncoderResourceManager::SetStartBitrate(
404     DataRate start_bitrate) {
405   RTC_DCHECK_RUN_ON(encoder_queue_);
406   if (!start_bitrate.IsZero()) {
407     encoder_target_bitrate_bps_ = start_bitrate.bps();
408     bitrate_constraint_->OnEncoderTargetBitrateUpdated(
409         encoder_target_bitrate_bps_);
410     balanced_constraint_->OnEncoderTargetBitrateUpdated(
411         encoder_target_bitrate_bps_);
412   }
413   initial_frame_dropper_->SetStartBitrate(start_bitrate,
414                                           clock_->TimeInMicroseconds());
415 }
416 
SetTargetBitrate(DataRate target_bitrate)417 void VideoStreamEncoderResourceManager::SetTargetBitrate(
418     DataRate target_bitrate) {
419   RTC_DCHECK_RUN_ON(encoder_queue_);
420   if (!target_bitrate.IsZero()) {
421     encoder_target_bitrate_bps_ = target_bitrate.bps();
422     bitrate_constraint_->OnEncoderTargetBitrateUpdated(
423         encoder_target_bitrate_bps_);
424     balanced_constraint_->OnEncoderTargetBitrateUpdated(
425         encoder_target_bitrate_bps_);
426   }
427   initial_frame_dropper_->SetTargetBitrate(target_bitrate,
428                                            clock_->TimeInMilliseconds());
429 }
430 
SetEncoderRates(const VideoEncoder::RateControlParameters & encoder_rates)431 void VideoStreamEncoderResourceManager::SetEncoderRates(
432     const VideoEncoder::RateControlParameters& encoder_rates) {
433   RTC_DCHECK_RUN_ON(encoder_queue_);
434   encoder_rates_ = encoder_rates;
435 }
436 
OnFrameDroppedDueToSize()437 void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
438   RTC_DCHECK_RUN_ON(encoder_queue_);
439   // The VideoStreamEncoder makes the manager outlive the adaptation queue. This
440   // means that if the task gets executed, |this| has not been freed yet.
441   // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives
442   // the adaptation queue, add logic to prevent use-after-free on |this|.
443   resource_adaptation_queue_->PostTask([this] {
444     RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
445     if (!adaptation_processor_) {
446       // The processor nulled before this task had a chance to execute. This
447       // happens if the processor is destroyed. No action needed.
448       return;
449     }
450     Adaptation reduce_resolution = stream_adapter_->GetAdaptDownResolution();
451     if (reduce_resolution.status() == Adaptation::Status::kValid) {
452       stream_adapter_->ApplyAdaptation(reduce_resolution,
453                                        quality_scaler_resource_);
454     }
455   });
456   initial_frame_dropper_->OnFrameDroppedDueToSize();
457 }
458 
OnEncodeStarted(const VideoFrame & cropped_frame,int64_t time_when_first_seen_us)459 void VideoStreamEncoderResourceManager::OnEncodeStarted(
460     const VideoFrame& cropped_frame,
461     int64_t time_when_first_seen_us) {
462   RTC_DCHECK_RUN_ON(encoder_queue_);
463   encode_usage_resource_->OnEncodeStarted(cropped_frame,
464                                           time_when_first_seen_us);
465 }
466 
OnEncodeCompleted(const EncodedImage & encoded_image,int64_t time_sent_in_us,absl::optional<int> encode_duration_us)467 void VideoStreamEncoderResourceManager::OnEncodeCompleted(
468     const EncodedImage& encoded_image,
469     int64_t time_sent_in_us,
470     absl::optional<int> encode_duration_us) {
471   RTC_DCHECK_RUN_ON(encoder_queue_);
472   // Inform |encode_usage_resource_| of the encode completed event.
473   uint32_t timestamp = encoded_image.Timestamp();
474   int64_t capture_time_us =
475       encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
476   encode_usage_resource_->OnEncodeCompleted(
477       timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
478   // Inform |quality_scaler_resource_| of the encode completed event.
479   quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
480 }
481 
OnFrameDropped(EncodedImageCallback::DropReason reason)482 void VideoStreamEncoderResourceManager::OnFrameDropped(
483     EncodedImageCallback::DropReason reason) {
484   RTC_DCHECK_RUN_ON(encoder_queue_);
485   quality_scaler_resource_->OnFrameDropped(reason);
486 }
487 
DropInitialFrames() const488 bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
489   RTC_DCHECK_RUN_ON(encoder_queue_);
490   return initial_frame_dropper_->DropInitialFrames();
491 }
492 
OnMaybeEncodeFrame()493 void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
494   RTC_DCHECK_RUN_ON(encoder_queue_);
495   initial_frame_dropper_->OnMaybeEncodeFrame();
496   if (quality_rampup_experiment_) {
497     DataRate bandwidth = encoder_rates_.has_value()
498                              ? encoder_rates_->bandwidth_allocation
499                              : DataRate::Zero();
500     quality_rampup_experiment_->PerformQualityRampupExperiment(
501         quality_scaler_resource_, bandwidth,
502         DataRate::BitsPerSec(encoder_target_bitrate_bps_.value_or(0)),
503         DataRate::KilobitsPerSec(encoder_settings_->video_codec().maxBitrate),
504         LastInputFrameSizeOrDefault());
505   }
506 }
507 
UpdateQualityScalerSettings(absl::optional<VideoEncoder::QpThresholds> qp_thresholds)508 void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
509     absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
510   RTC_DCHECK_RUN_ON(encoder_queue_);
511   if (qp_thresholds.has_value()) {
512     quality_scaler_resource_->StopCheckForOveruse();
513     quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
514   } else {
515     quality_scaler_resource_->StopCheckForOveruse();
516   }
517   initial_frame_dropper_->OnQualityScalerSettingsUpdated();
518 }
519 
ConfigureQualityScaler(const VideoEncoder::EncoderInfo & encoder_info)520 void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
521     const VideoEncoder::EncoderInfo& encoder_info) {
522   RTC_DCHECK_RUN_ON(encoder_queue_);
523   const auto scaling_settings = encoder_info.scaling_settings;
524   const bool quality_scaling_allowed =
525       IsResolutionScalingEnabled(degradation_preference_) &&
526       scaling_settings.thresholds;
527 
528   // TODO(https://crbug.com/webrtc/11222): Should this move to
529   // QualityScalerResource?
530   if (quality_scaling_allowed) {
531     if (!quality_scaler_resource_->is_started()) {
532       // Quality scaler has not already been configured.
533 
534       // Use experimental thresholds if available.
535       absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
536       if (quality_scaling_experiment_enabled_) {
537         experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
538             GetVideoCodecTypeOrGeneric(encoder_settings_));
539       }
540       UpdateQualityScalerSettings(experimental_thresholds
541                                       ? *experimental_thresholds
542                                       : *(scaling_settings.thresholds));
543     }
544   } else {
545     UpdateQualityScalerSettings(absl::nullopt);
546   }
547 
548   // Set the qp-thresholds to the balanced settings if balanced mode.
549   if (degradation_preference_ == DegradationPreference::BALANCED &&
550       quality_scaler_resource_->is_started()) {
551     absl::optional<VideoEncoder::QpThresholds> thresholds =
552         balanced_settings_.GetQpThresholds(
553             GetVideoCodecTypeOrGeneric(encoder_settings_),
554             LastInputFrameSizeOrDefault());
555     if (thresholds) {
556       quality_scaler_resource_->SetQpThresholds(*thresholds);
557     }
558   }
559   UpdateStatsAdaptationSettings();
560 }
561 
GetReasonFromResource(rtc::scoped_refptr<Resource> resource) const562 VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
563     rtc::scoped_refptr<Resource> resource) const {
564   MutexLock lock(&resource_lock_);
565   const auto& registered_resource =
566       absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
567         return r.resource == resource;
568       });
569   RTC_DCHECK(registered_resource != resources_.end())
570       << resource->Name() << " not found.";
571   return registered_resource->reason;
572 }
573 
574 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
575 // pipelining encoders better (multiple input frames before something comes
576 // out). This should effectively turn off CPU adaptations for systems that
577 // remotely cope with the load right now.
GetCpuOveruseOptions() const578 CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions()
579     const {
580   RTC_DCHECK_RUN_ON(encoder_queue_);
581   // This is already ensured by the only caller of this method:
582   // StartResourceAdaptation().
583   RTC_DCHECK(encoder_settings_.has_value());
584   CpuOveruseOptions options;
585   // Hardware accelerated encoders are assumed to be pipelined; give them
586   // additional overuse time.
587   if (encoder_settings_->encoder_info().is_hardware_accelerated) {
588     options.low_encode_usage_threshold_percent = 150;
589     options.high_encode_usage_threshold_percent = 200;
590   }
591   if (experiment_cpu_load_estimator_) {
592     options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
593   }
594   return options;
595 }
596 
LastInputFrameSizeOrDefault() const597 int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const {
598   RTC_DCHECK_RUN_ON(encoder_queue_);
599   return input_state_provider_->InputState().frame_size_pixels().value_or(
600       kDefaultInputPixelsWidth * kDefaultInputPixelsHeight);
601 }
602 
OnVideoSourceRestrictionsUpdated(VideoSourceRestrictions restrictions,const VideoAdaptationCounters & adaptation_counters,rtc::scoped_refptr<Resource> reason,const VideoSourceRestrictions & unfiltered_restrictions)603 void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
604     VideoSourceRestrictions restrictions,
605     const VideoAdaptationCounters& adaptation_counters,
606     rtc::scoped_refptr<Resource> reason,
607     const VideoSourceRestrictions& unfiltered_restrictions) {
608   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
609   // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
610   if (!reason && adaptation_counters.Total() == 0) {
611     // Adaptation was manually reset - clear the per-reason counters too.
612     encoder_stats_observer_->ClearAdaptationStats();
613   }
614 
615   // The VideoStreamEncoder makes the manager outlive the encoder queue. This
616   // means that if the task gets executed, |this| has not been freed yet.
617   encoder_queue_->PostTask([this, restrictions] {
618     RTC_DCHECK_RUN_ON(encoder_queue_);
619     video_source_restrictions_ = FilterRestrictionsByDegradationPreference(
620         restrictions, degradation_preference_);
621     MaybeUpdateTargetFrameRate();
622   });
623 }
624 
OnResourceLimitationChanged(rtc::scoped_refptr<Resource> resource,const std::map<rtc::scoped_refptr<Resource>,VideoAdaptationCounters> & resource_limitations)625 void VideoStreamEncoderResourceManager::OnResourceLimitationChanged(
626     rtc::scoped_refptr<Resource> resource,
627     const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
628         resource_limitations) {
629   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
630   if (!resource) {
631     encoder_stats_observer_->ClearAdaptationStats();
632     return;
633   }
634 
635   std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations;
636   for (auto& resource_counter : resource_limitations) {
637     std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it;
638     bool inserted;
639     std::tie(it, inserted) = limitations.emplace(
640         GetReasonFromResource(resource_counter.first), resource_counter.second);
641     if (!inserted && it->second.Total() < resource_counter.second.Total()) {
642       it->second = resource_counter.second;
643     }
644   }
645 
646   VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource);
647   encoder_stats_observer_->OnAdaptationChanged(
648       adaptation_reason, limitations[VideoAdaptationReason::kCpu],
649       limitations[VideoAdaptationReason::kQuality]);
650 
651   encoder_queue_->PostTask(ToQueuedTask(
652       [cpu_limited = limitations.at(VideoAdaptationReason::kCpu).Total() > 0,
653        qp_resolution_adaptations =
654            limitations.at(VideoAdaptationReason::kQuality)
655                .resolution_adaptations,
656        this]() {
657         RTC_DCHECK_RUN_ON(encoder_queue_);
658         if (quality_rampup_experiment_) {
659           quality_rampup_experiment_->cpu_adapted(cpu_limited);
660           quality_rampup_experiment_->qp_resolution_adaptations(
661               qp_resolution_adaptations);
662         }
663       }));
664 
665   RTC_LOG(LS_INFO) << ActiveCountsToString(limitations);
666 }
667 
MaybeUpdateTargetFrameRate()668 void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
669   RTC_DCHECK_RUN_ON(encoder_queue_);
670   absl::optional<double> codec_max_frame_rate =
671       encoder_settings_.has_value()
672           ? absl::optional<double>(
673                 encoder_settings_->video_codec().maxFramerate)
674           : absl::nullopt;
675   // The current target framerate is the maximum frame rate as specified by
676   // the current codec configuration or any limit imposed by the adaptation
677   // module. This is used to make sure overuse detection doesn't needlessly
678   // trigger in low and/or variable framerate scenarios.
679   absl::optional<double> target_frame_rate =
680       video_source_restrictions_.max_frame_rate();
681   if (!target_frame_rate.has_value() ||
682       (codec_max_frame_rate.has_value() &&
683        codec_max_frame_rate.value() < target_frame_rate.value())) {
684     target_frame_rate = codec_max_frame_rate;
685   }
686   encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
687 }
688 
UpdateStatsAdaptationSettings() const689 void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
690   RTC_DCHECK_RUN_ON(encoder_queue_);
691   VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
692       IsResolutionScalingEnabled(degradation_preference_),
693       IsFramerateScalingEnabled(degradation_preference_));
694 
695   VideoStreamEncoderObserver::AdaptationSettings quality_settings =
696       quality_scaler_resource_->is_started()
697           ? cpu_settings
698           : VideoStreamEncoderObserver::AdaptationSettings();
699   encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
700                                                     quality_settings);
701 }
702 
703 // static
ActiveCountsToString(const std::map<VideoAdaptationReason,VideoAdaptationCounters> & active_counts)704 std::string VideoStreamEncoderResourceManager::ActiveCountsToString(
705     const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
706         active_counts) {
707   rtc::StringBuilder ss;
708 
709   ss << "Downgrade counts: fps: {";
710   for (auto& reason_count : active_counts) {
711     ss << ToString(reason_count.first) << ":";
712     ss << reason_count.second.fps_adaptations;
713   }
714   ss << "}, resolution {";
715   for (auto& reason_count : active_counts) {
716     ss << ToString(reason_count.first) << ":";
717     ss << reason_count.second.resolution_adaptations;
718   }
719   ss << "}";
720 
721   return ss.Release();
722 }
723 
OnQualityRampUp()724 void VideoStreamEncoderResourceManager::OnQualityRampUp() {
725   RTC_DCHECK_RUN_ON(encoder_queue_);
726   // The VideoStreamEncoder makes the manager outlive the adaptation queue.
727   // This means that if the task gets executed, |this| has not been freed yet.
728   // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives
729   // the adaptation queue, add logic to prevent use-after-free on |this|.
730   resource_adaptation_queue_->PostTask([this] {
731     RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
732     if (!stream_adapter_) {
733       // The processor nulled before this task had a chance to execute. This
734       // happens if the processor is destroyed. No action needed.
735       return;
736     }
737     stream_adapter_->ClearRestrictions();
738   });
739   quality_rampup_experiment_.reset();
740 }
741 }  // namespace webrtc
742