1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
12 #define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
13 
14 #include <list>
15 
16 #include "webrtc/base/scoped_ptr.h"
17 #include "webrtc/modules/include/module_common_types.h"
18 #include "webrtc/modules/video_coding/include/video_coding.h"
19 #include "webrtc/modules/video_coding/media_opt_util.h"
20 #include "webrtc/modules/video_coding/qm_select.h"
21 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
22 
23 namespace webrtc {
24 
25 // Forward declarations.
26 class Clock;
27 class FrameDropper;
28 class VCMContentMetricsProcessing;
29 
30 namespace media_optimization {
31 
32 class MediaOptimization {
33  public:
34   explicit MediaOptimization(Clock* clock);
35   ~MediaOptimization();
36 
37   // TODO(andresp): Can Reset and SetEncodingData be done at construction time
38   // only?
39   void Reset();
40 
41   // Informs media optimization of initial encoding state.
42   void SetEncodingData(VideoCodecType send_codec_type,
43                        int32_t max_bit_rate,
44                        uint32_t bit_rate,
45                        uint16_t width,
46                        uint16_t height,
47                        uint32_t frame_rate,
48                        int num_temporal_layers,
49                        int32_t mtu);
50 
51   // Sets target rates for the encoder given the channel parameters.
52   // Inputs:  target bitrate - the encoder target bitrate in bits/s.
53   //          fraction_lost - packet loss rate in % in the network.
54   //          round_trip_time_ms - round trip time in milliseconds.
55   //          min_bit_rate - the bit rate of the end-point with lowest rate.
56   //          max_bit_rate - the bit rate of the end-point with highest rate.
57   // TODO(andresp): Find if the callbacks can be triggered only after releasing
58   // an internal critical section.
59   uint32_t SetTargetRates(uint32_t target_bitrate,
60                           uint8_t fraction_lost,
61                           int64_t round_trip_time_ms,
62                           VCMProtectionCallback* protection_callback,
63                           VCMQMSettingsCallback* qmsettings_callback);
64 
65   void SetProtectionMethod(VCMProtectionMethodEnum method);
66   void EnableQM(bool enable);
67   void EnableFrameDropper(bool enable);
68 
69   // Lets the sender suspend video when the rate drops below
70   // |threshold_bps|, and turns back on when the rate goes back up above
71   // |threshold_bps| + |window_bps|.
72   void SuspendBelowMinBitrate(int threshold_bps, int window_bps);
73   bool IsVideoSuspended() const;
74 
75   bool DropFrame();
76 
77   void UpdateContentData(const VideoContentMetrics* content_metrics);
78 
79   // Informs Media Optimization of encoded output.
80   int32_t UpdateWithEncodedData(const EncodedImage& encoded_image);
81 
82   // InputFrameRate 0 = no frame rate estimate available.
83   uint32_t InputFrameRate();
84   uint32_t SentFrameRate();
85   uint32_t SentBitRate();
86 
87  private:
88   enum { kFrameCountHistorySize = 90 };
89   enum { kFrameHistoryWinMs = 2000 };
90   enum { kBitrateAverageWinMs = 1000 };
91 
92   struct EncodedFrameSample;
93   typedef std::list<EncodedFrameSample> FrameSampleList;
94 
95   void UpdateIncomingFrameRate() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
96   void PurgeOldFrameSamples(int64_t now_ms)
97       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
98   void UpdateSentBitrate(int64_t now_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
99   void UpdateSentFramerate() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
100 
101   // Computes new Quality Mode.
102   int32_t SelectQuality(VCMQMSettingsCallback* qmsettings_callback)
103       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
104 
105   // Verifies if QM settings differ from default, i.e. if an update is required.
106   // Computes actual values, as will be sent to the encoder.
107   bool QMUpdate(VCMResolutionScale* qm,
108                 VCMQMSettingsCallback* qmsettings_callback)
109       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
110 
111   // Checks if we should make a QM change. Return true if yes, false otherwise.
112   bool CheckStatusForQMchange() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
113 
114   void ProcessIncomingFrameRate(int64_t now)
115       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
116 
117   // Checks conditions for suspending the video. The method compares
118   // |video_target_bitrate_| with the threshold values for suspension, and
119   // changes the state of |video_suspended_| accordingly.
120   void CheckSuspendConditions() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
121 
122   void SetEncodingDataInternal(VideoCodecType send_codec_type,
123                                int32_t max_bit_rate,
124                                uint32_t frame_rate,
125                                uint32_t bit_rate,
126                                uint16_t width,
127                                uint16_t height,
128                                int num_temporal_layers,
129                                int32_t mtu)
130       EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
131 
132   uint32_t InputFrameRateInternal() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
133 
134   uint32_t SentFrameRateInternal() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
135 
136   // Protect all members.
137   rtc::scoped_ptr<CriticalSectionWrapper> crit_sect_;
138 
139   Clock* clock_ GUARDED_BY(crit_sect_);
140   int32_t max_bit_rate_ GUARDED_BY(crit_sect_);
141   VideoCodecType send_codec_type_ GUARDED_BY(crit_sect_);
142   uint16_t codec_width_ GUARDED_BY(crit_sect_);
143   uint16_t codec_height_ GUARDED_BY(crit_sect_);
144   float user_frame_rate_ GUARDED_BY(crit_sect_);
145   rtc::scoped_ptr<FrameDropper> frame_dropper_ GUARDED_BY(crit_sect_);
146   rtc::scoped_ptr<VCMLossProtectionLogic> loss_prot_logic_
147       GUARDED_BY(crit_sect_);
148   uint8_t fraction_lost_ GUARDED_BY(crit_sect_);
149   uint32_t send_statistics_[4] GUARDED_BY(crit_sect_);
150   uint32_t send_statistics_zero_encode_ GUARDED_BY(crit_sect_);
151   int32_t max_payload_size_ GUARDED_BY(crit_sect_);
152   int video_target_bitrate_ GUARDED_BY(crit_sect_);
153   float incoming_frame_rate_ GUARDED_BY(crit_sect_);
154   int64_t incoming_frame_times_[kFrameCountHistorySize] GUARDED_BY(crit_sect_);
155   bool enable_qm_ GUARDED_BY(crit_sect_);
156   std::list<EncodedFrameSample> encoded_frame_samples_ GUARDED_BY(crit_sect_);
157   uint32_t avg_sent_bit_rate_bps_ GUARDED_BY(crit_sect_);
158   uint32_t avg_sent_framerate_ GUARDED_BY(crit_sect_);
159   uint32_t key_frame_cnt_ GUARDED_BY(crit_sect_);
160   uint32_t delta_frame_cnt_ GUARDED_BY(crit_sect_);
161   rtc::scoped_ptr<VCMContentMetricsProcessing> content_ GUARDED_BY(crit_sect_);
162   rtc::scoped_ptr<VCMQmResolution> qm_resolution_ GUARDED_BY(crit_sect_);
163   int64_t last_qm_update_time_ GUARDED_BY(crit_sect_);
164   int64_t last_change_time_ GUARDED_BY(crit_sect_);  // Content/user triggered.
165   int num_layers_ GUARDED_BY(crit_sect_);
166   bool suspension_enabled_ GUARDED_BY(crit_sect_);
167   bool video_suspended_ GUARDED_BY(crit_sect_);
168   int suspension_threshold_bps_ GUARDED_BY(crit_sect_);
169   int suspension_window_bps_ GUARDED_BY(crit_sect_);
170 };
171 }  // namespace media_optimization
172 }  // namespace webrtc
173 
174 #endif  // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
175