1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
12 #define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
13 
14 #include <math.h>
15 #include <stdlib.h>
16 
17 #include "webrtc/base/exp_filter.h"
18 #include "webrtc/base/scoped_ptr.h"
19 #include "webrtc/modules/video_coding/internal_defines.h"
20 #include "webrtc/modules/video_coding/qm_select.h"
21 #include "webrtc/system_wrappers/include/trace.h"
22 #include "webrtc/typedefs.h"
23 
24 namespace webrtc {
25 namespace media_optimization {
26 
27 // Number of time periods used for (max) window filter for packet loss
28 // TODO(marpan): set reasonable window size for filtered packet loss,
29 // adjustment should be based on logged/real data of loss stats/correlation.
30 enum { kLossPrHistorySize = 10 };
31 
32 // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
33 enum { kLossPrShortFilterWinMs = 1000 };
34 
35 // The type of filter used on the received packet loss reports.
36 enum FilterPacketLossMode {
37   kNoFilter,   // No filtering on received loss.
38   kAvgFilter,  // Recursive average filter.
39   kMaxFilter   // Max-window filter, over the time interval of:
40                // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
41 };
42 
43 // Thresholds for hybrid NACK/FEC
44 // common to media optimization and the jitter buffer.
45 const int64_t kLowRttNackMs = 20;
46 
47 struct VCMProtectionParameters {
VCMProtectionParametersVCMProtectionParameters48   VCMProtectionParameters()
49       : rtt(0),
50         lossPr(0.0f),
51         bitRate(0.0f),
52         packetsPerFrame(0.0f),
53         packetsPerFrameKey(0.0f),
54         frameRate(0.0f),
55         keyFrameSize(0.0f),
56         fecRateDelta(0),
57         fecRateKey(0),
58         codecWidth(0),
59         codecHeight(0),
60         numLayers(1) {}
61 
62   int64_t rtt;
63   float lossPr;
64   float bitRate;
65   float packetsPerFrame;
66   float packetsPerFrameKey;
67   float frameRate;
68   float keyFrameSize;
69   uint8_t fecRateDelta;
70   uint8_t fecRateKey;
71   uint16_t codecWidth;
72   uint16_t codecHeight;
73   int numLayers;
74 };
75 
76 /******************************/
77 /* VCMProtectionMethod class  */
78 /******************************/
79 
80 enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
81 
82 class VCMLossProbabilitySample {
83  public:
VCMLossProbabilitySample()84   VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
85 
86   uint8_t lossPr255;
87   int64_t timeMs;
88 };
89 
90 class VCMProtectionMethod {
91  public:
92   VCMProtectionMethod();
93   virtual ~VCMProtectionMethod();
94 
95   // Updates the efficiency of the method using the parameters provided
96   //
97   // Input:
98   //         - parameters         : Parameters used to calculate efficiency
99   //
100   // Return value                 : True if this method is recommended in
101   //                                the given conditions.
102   virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
103 
104   // Returns the protection type
105   //
106   // Return value                 : The protection type
Type()107   enum VCMProtectionMethodEnum Type() const { return _type; }
108 
109   // Returns the effective packet loss for ER, required by this protection
110   // method
111   //
112   // Return value                 : Required effective packet loss
RequiredPacketLossER()113   virtual uint8_t RequiredPacketLossER() { return _effectivePacketLoss; }
114 
115   // Extracts the FEC protection factor for Key frame, required by this
116   // protection method
117   //
118   // Return value                 : Required protectionFactor for Key frame
RequiredProtectionFactorK()119   virtual uint8_t RequiredProtectionFactorK() { return _protectionFactorK; }
120 
121   // Extracts the FEC protection factor for Delta frame, required by this
122   // protection method
123   //
124   // Return value                 : Required protectionFactor for delta frame
RequiredProtectionFactorD()125   virtual uint8_t RequiredProtectionFactorD() { return _protectionFactorD; }
126 
127   // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
128   //
129   // Return value                 : Required Unequal protection on/off state.
RequiredUepProtectionK()130   virtual bool RequiredUepProtectionK() { return _useUepProtectionK; }
131 
132   // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
133   // frame.
134   //
135   // Return value                 : Required Unequal protection on/off state.
RequiredUepProtectionD()136   virtual bool RequiredUepProtectionD() { return _useUepProtectionD; }
137 
MaxFramesFec()138   virtual int MaxFramesFec() const { return 1; }
139 
140   // Updates content metrics
141   void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
142 
143  protected:
144   uint8_t _effectivePacketLoss;
145   uint8_t _protectionFactorK;
146   uint8_t _protectionFactorD;
147   // Estimation of residual loss after the FEC
148   float _scaleProtKey;
149   int32_t _maxPayloadSize;
150 
151   VCMQmRobustness* _qmRobustness;
152   bool _useUepProtectionK;
153   bool _useUepProtectionD;
154   float _corrFecCost;
155   enum VCMProtectionMethodEnum _type;
156 };
157 
158 class VCMNackMethod : public VCMProtectionMethod {
159  public:
160   VCMNackMethod();
161   virtual ~VCMNackMethod();
162   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
163   // Get the effective packet loss
164   bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
165 };
166 
167 class VCMFecMethod : public VCMProtectionMethod {
168  public:
169   VCMFecMethod();
170   virtual ~VCMFecMethod();
171   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
172   // Get the effective packet loss for ER
173   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
174   // Get the FEC protection factors
175   bool ProtectionFactor(const VCMProtectionParameters* parameters);
176   // Get the boost for key frame protection
177   uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
178                            uint8_t packetFrameKey) const;
179   // Convert the rates: defined relative to total# packets or source# packets
180   uint8_t ConvertFECRate(uint8_t codeRate) const;
181   // Get the average effective recovery from FEC: for random loss model
182   float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
183   // Update FEC with protectionFactorD
184   void UpdateProtectionFactorD(uint8_t protectionFactorD);
185   // Update FEC with protectionFactorK
186   void UpdateProtectionFactorK(uint8_t protectionFactorK);
187   // Compute the bits per frame. Account for temporal layers when applicable.
188   int BitsPerFrame(const VCMProtectionParameters* parameters);
189 
190  protected:
191   enum { kUpperLimitFramesFec = 6 };
192   // Thresholds values for the bytes/frame and round trip time, below which we
193   // may turn off FEC, depending on |_numLayers| and |_maxFramesFec|.
194   // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
195   enum { kMaxBytesPerFrameForFec = 700 };
196   // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
197   enum { kMaxBytesPerFrameForFecLow = 400 };
198   // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
199   enum { kMaxBytesPerFrameForFecHigh = 1000 };
200 };
201 
202 class VCMNackFecMethod : public VCMFecMethod {
203  public:
204   VCMNackFecMethod(int64_t lowRttNackThresholdMs,
205                    int64_t highRttNackThresholdMs);
206   virtual ~VCMNackFecMethod();
207   virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
208   // Get the effective packet loss for ER
209   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
210   // Get the protection factors
211   bool ProtectionFactor(const VCMProtectionParameters* parameters);
212   // Get the max number of frames the FEC is allowed to be based on.
213   int MaxFramesFec() const;
214   // Turn off the FEC based on low bitrate and other factors.
215   bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
216 
217  private:
218   int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
219 
220   int64_t _lowRttNackMs;
221   int64_t _highRttNackMs;
222   int _maxFramesFec;
223 };
224 
225 class VCMLossProtectionLogic {
226  public:
227   explicit VCMLossProtectionLogic(int64_t nowMs);
228   ~VCMLossProtectionLogic();
229 
230   // Set the protection method to be used
231   //
232   // Input:
233   //        - newMethodType    : New requested protection method type. If one
234   //                           is already set, it will be deleted and replaced
235   void SetMethod(VCMProtectionMethodEnum newMethodType);
236 
237   // Update the round-trip time
238   //
239   // Input:
240   //          - rtt           : Round-trip time in seconds.
241   void UpdateRtt(int64_t rtt);
242 
243   // Update the filtered packet loss.
244   //
245   // Input:
246   //          - packetLossEnc :  The reported packet loss filtered
247   //                             (max window or average)
248   void UpdateFilteredLossPr(uint8_t packetLossEnc);
249 
250   // Update the current target bit rate.
251   //
252   // Input:
253   //          - bitRate          : The current target bit rate in kbits/s
254   void UpdateBitRate(float bitRate);
255 
256   // Update the number of packets per frame estimate, for delta frames
257   //
258   // Input:
259   //          - nPackets         : Number of packets in the latest sent frame.
260   void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
261 
262   // Update the number of packets per frame estimate, for key frames
263   //
264   // Input:
265   //          - nPackets         : umber of packets in the latest sent frame.
266   void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
267 
268   // Update the keyFrameSize estimate
269   //
270   // Input:
271   //          - keyFrameSize     : The size of the latest sent key frame.
272   void UpdateKeyFrameSize(float keyFrameSize);
273 
274   // Update the frame rate
275   //
276   // Input:
277   //          - frameRate        : The current target frame rate.
UpdateFrameRate(float frameRate)278   void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
279 
280   // Update the frame size
281   //
282   // Input:
283   //          - width        : The codec frame width.
284   //          - height       : The codec frame height.
285   void UpdateFrameSize(uint16_t width, uint16_t height);
286 
287   // Update the number of active layers
288   //
289   // Input:
290   //          - numLayers    : Number of layers used.
291   void UpdateNumLayers(int numLayers);
292 
293   // The amount of packet loss to cover for with FEC.
294   //
295   // Input:
296   //          - fecRateKey      : Packet loss to cover for with FEC when
297   //                              sending key frames.
298   //          - fecRateDelta    : Packet loss to cover for with FEC when
299   //                              sending delta frames.
UpdateFECRates(uint8_t fecRateKey,uint8_t fecRateDelta)300   void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
301     _fecRateKey = fecRateKey;
302     _fecRateDelta = fecRateDelta;
303   }
304 
305   // Update the protection methods with the current VCMProtectionParameters
306   // and set the requested protection settings.
307   // Return value     : Returns true on update
308   bool UpdateMethod();
309 
310   // Returns the method currently selected.
311   //
312   // Return value                 : The protection method currently selected.
313   VCMProtectionMethod* SelectedMethod() const;
314 
315   // Return the protection type of the currently selected method
316   VCMProtectionMethodEnum SelectedType() const;
317 
318   // Updates the filtered loss for the average and max window packet loss,
319   // and returns the filtered loss probability in the interval [0, 255].
320   // The returned filtered loss value depends on the parameter |filter_mode|.
321   // The input parameter |lossPr255| is the received packet loss.
322 
323   // Return value                 : The filtered loss probability
324   uint8_t FilteredLoss(int64_t nowMs,
325                        FilterPacketLossMode filter_mode,
326                        uint8_t lossPr255);
327 
328   void Reset(int64_t nowMs);
329 
330   void Release();
331 
332  private:
333   // Sets the available loss protection methods.
334   void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
335   uint8_t MaxFilteredLossPr(int64_t nowMs) const;
336   rtc::scoped_ptr<VCMProtectionMethod> _selectedMethod;
337   VCMProtectionParameters _currentParameters;
338   int64_t _rtt;
339   float _lossPr;
340   float _bitRate;
341   float _frameRate;
342   float _keyFrameSize;
343   uint8_t _fecRateKey;
344   uint8_t _fecRateDelta;
345   int64_t _lastPrUpdateT;
346   int64_t _lastPacketPerFrameUpdateT;
347   int64_t _lastPacketPerFrameUpdateTKey;
348   rtc::ExpFilter _lossPr255;
349   VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
350   uint8_t _shortMaxLossPr255;
351   rtc::ExpFilter _packetsPerFrame;
352   rtc::ExpFilter _packetsPerFrameKey;
353   uint16_t _codecWidth;
354   uint16_t _codecHeight;
355   int _numLayers;
356 };
357 
358 }  // namespace media_optimization
359 }  // namespace webrtc
360 
361 #endif  // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
362