1 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
2 *
3 * Use of this source code is governed by a BSD-style license
4 * that can be found in the LICENSE file in the root of the source
5 * tree. An additional intellectual property rights grant can be found
6 * in the file PATENTS. All contributing project authors may
7 * be found in the AUTHORS file in the root of the source tree.
8 */
9
10 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
11
12 #include <stdlib.h>
13
14 #include <algorithm>
15
16 #include "webrtc/base/checks.h"
17 #include "vpx/vpx_encoder.h"
18 #include "vpx/vp8cx.h"
19 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
20
21 namespace webrtc {
22
23 static const int kOneSecond90Khz = 90000;
24 static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
25 static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
26 static const int kQpDeltaThresholdForSync = 8;
27
28 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
29 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
30
31 // Since this is TL0 we only allow updating and predicting from the LAST
32 // reference frame.
33 const int ScreenshareLayers::kTl0Flags =
34 VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
35 VP8_EFLAG_NO_REF_ARF;
36
37 // Allow predicting from both TL0 and TL1.
38 const int ScreenshareLayers::kTl1Flags =
39 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
40
41 // Allow predicting from only TL0 to allow participants to switch to the high
42 // bitrate stream. This means predicting only from the LAST reference frame, but
43 // only updating GF to not corrupt TL0.
44 const int ScreenshareLayers::kTl1SyncFlags =
45 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
46 VP8_EFLAG_NO_UPD_LAST;
47
ScreenshareLayers(int num_temporal_layers,uint8_t initial_tl0_pic_idx)48 ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
49 uint8_t initial_tl0_pic_idx)
50 : number_of_temporal_layers_(num_temporal_layers),
51 last_base_layer_sync_(false),
52 tl0_pic_idx_(initial_tl0_pic_idx),
53 active_layer_(-1),
54 last_timestamp_(-1),
55 last_sync_timestamp_(-1),
56 min_qp_(-1),
57 max_qp_(-1),
58 max_debt_bytes_(0),
59 frame_rate_(-1) {
60 assert(num_temporal_layers > 0);
61 assert(num_temporal_layers <= 2);
62 }
63
CurrentLayerId() const64 int ScreenshareLayers::CurrentLayerId() const {
65 // Codec does not use temporal layers for screenshare.
66 return 0;
67 }
68
EncodeFlags(uint32_t timestamp)69 int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
70 if (number_of_temporal_layers_ <= 1) {
71 // No flags needed for 1 layer screenshare.
72 return 0;
73 }
74
75 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
76 int flags = 0;
77
78 if (active_layer_ == -1 ||
79 layers_[active_layer_].state != TemporalLayer::State::kDropped) {
80 if (layers_[0].debt_bytes_ > max_debt_bytes_) {
81 // Must drop TL0, encode TL1 instead.
82 if (layers_[1].debt_bytes_ > max_debt_bytes_) {
83 // Must drop both TL0 and TL1.
84 active_layer_ = -1;
85 } else {
86 active_layer_ = 1;
87 }
88 } else {
89 active_layer_ = 0;
90 }
91 }
92
93 switch (active_layer_) {
94 case 0:
95 flags = kTl0Flags;
96 break;
97 case 1:
98 if (TimeToSync(unwrapped_timestamp)) {
99 last_sync_timestamp_ = unwrapped_timestamp;
100 flags = kTl1SyncFlags;
101 } else {
102 flags = kTl1Flags;
103 }
104 break;
105 case -1:
106 flags = -1;
107 break;
108 default:
109 flags = -1;
110 RTC_NOTREACHED();
111 }
112
113 // Make sure both frame droppers leak out bits.
114 int64_t ts_diff;
115 if (last_timestamp_ == -1) {
116 ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_);
117 } else {
118 ts_diff = unwrapped_timestamp - last_timestamp_;
119 }
120
121 layers_[0].UpdateDebt(ts_diff / 90);
122 layers_[1].UpdateDebt(ts_diff / 90);
123 last_timestamp_ = timestamp;
124 return flags;
125 }
126
ConfigureBitrates(int bitrate_kbps,int max_bitrate_kbps,int framerate,vpx_codec_enc_cfg_t * cfg)127 bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps,
128 int max_bitrate_kbps,
129 int framerate,
130 vpx_codec_enc_cfg_t* cfg) {
131 layers_[0].target_rate_kbps_ = bitrate_kbps;
132 layers_[1].target_rate_kbps_ = max_bitrate_kbps;
133
134 int target_bitrate_kbps = bitrate_kbps;
135
136 if (cfg != nullptr) {
137 if (number_of_temporal_layers_ > 1) {
138 // Calculate a codec target bitrate. This may be higher than TL0, gaining
139 // quality at the expense of frame rate at TL0. Constraints:
140 // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
141 // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
142 target_bitrate_kbps =
143 std::min(bitrate_kbps * kMaxTL0FpsReduction,
144 max_bitrate_kbps / kAcceptableTargetOvershoot);
145
146 cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps);
147 }
148
149 // Don't reconfigure qp limits during quality boost frames.
150 if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
151 min_qp_ = cfg->rc_min_quantizer;
152 max_qp_ = cfg->rc_max_quantizer;
153 // After a dropped frame, a frame with max qp will be encoded and the
154 // quality will then ramp up from there. To boost the speed of recovery,
155 // encode the next frame with lower max qp. TL0 is the most important to
156 // improve since the errors in this layer will propagate to TL1.
157 // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
158 layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
159 layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
160 }
161 }
162
163 int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate);
164 max_debt_bytes_ = 4 * avg_frame_size;
165
166 return true;
167 }
168
FrameEncoded(unsigned int size,uint32_t timestamp,int qp)169 void ScreenshareLayers::FrameEncoded(unsigned int size,
170 uint32_t timestamp,
171 int qp) {
172 if (size == 0) {
173 layers_[active_layer_].state = TemporalLayer::State::kDropped;
174 return;
175 }
176
177 if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
178 layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
179 }
180
181 if (qp != -1)
182 layers_[active_layer_].last_qp = qp;
183
184 if (active_layer_ == 0) {
185 layers_[0].debt_bytes_ += size;
186 layers_[1].debt_bytes_ += size;
187 } else if (active_layer_ == 1) {
188 layers_[1].debt_bytes_ += size;
189 }
190 }
191
PopulateCodecSpecific(bool base_layer_sync,CodecSpecificInfoVP8 * vp8_info,uint32_t timestamp)192 void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
193 CodecSpecificInfoVP8* vp8_info,
194 uint32_t timestamp) {
195 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
196 if (number_of_temporal_layers_ == 1) {
197 vp8_info->temporalIdx = kNoTemporalIdx;
198 vp8_info->layerSync = false;
199 vp8_info->tl0PicIdx = kNoTl0PicIdx;
200 } else {
201 vp8_info->temporalIdx = active_layer_;
202 if (base_layer_sync) {
203 vp8_info->temporalIdx = 0;
204 last_sync_timestamp_ = unwrapped_timestamp;
205 } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
206 // Regardless of pattern the frame after a base layer sync will always
207 // be a layer sync.
208 last_sync_timestamp_ = unwrapped_timestamp;
209 }
210 vp8_info->layerSync = last_sync_timestamp_ != -1 &&
211 last_sync_timestamp_ == unwrapped_timestamp;
212 if (vp8_info->temporalIdx == 0) {
213 tl0_pic_idx_++;
214 }
215 last_base_layer_sync_ = base_layer_sync;
216 vp8_info->tl0PicIdx = tl0_pic_idx_;
217 }
218 }
219
TimeToSync(int64_t timestamp) const220 bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
221 if (active_layer_ != 1) {
222 RTC_NOTREACHED();
223 return false;
224 }
225 RTC_DCHECK_NE(-1, layers_[0].last_qp);
226 if (layers_[1].last_qp == -1) {
227 // First frame in TL1 should only depend on TL0 since there are no
228 // previous frames in TL1.
229 return true;
230 }
231
232 RTC_DCHECK_NE(-1, last_sync_timestamp_);
233 int64_t timestamp_diff = timestamp - last_sync_timestamp_;
234 if (timestamp_diff > kMaxTimeBetweenSyncs) {
235 // After a certain time, force a sync frame.
236 return true;
237 } else if (timestamp_diff < kMinTimeBetweenSyncs) {
238 // If too soon from previous sync frame, don't issue a new one.
239 return false;
240 }
241 // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
242 // large.
243 if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
244 return true;
245 return false;
246 }
247
UpdateConfiguration(vpx_codec_enc_cfg_t * cfg)248 bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
249 if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
250 return false;
251
252 // If layer is in the quality boost state (following a dropped frame), update
253 // the configuration with the adjusted (lower) qp and set the state back to
254 // normal.
255 unsigned int adjusted_max_qp;
256 if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
257 layers_[active_layer_].enhanced_max_qp != -1) {
258 adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
259 layers_[active_layer_].state = TemporalLayer::State::kNormal;
260 } else {
261 if (max_qp_ == -1)
262 return false;
263 adjusted_max_qp = max_qp_; // Set the normal max qp.
264 }
265
266 if (adjusted_max_qp == cfg->rc_max_quantizer)
267 return false;
268
269 cfg->rc_max_quantizer = adjusted_max_qp;
270 return true;
271 }
272
UpdateDebt(int64_t delta_ms)273 void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
274 uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
275 if (debt_reduction_bytes >= debt_bytes_) {
276 debt_bytes_ = 0;
277 } else {
278 debt_bytes_ -= debt_reduction_bytes;
279 }
280 }
281
282 } // namespace webrtc
283