1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/include/video_codec_initializer.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15
16 #include <memory>
17
18 #include "absl/types/optional.h"
19 #include "api/scoped_refptr.h"
20 #include "api/test/mock_fec_controller_override.h"
21 #include "api/video/builtin_video_bitrate_allocator_factory.h"
22 #include "api/video/video_bitrate_allocation.h"
23 #include "api/video/video_bitrate_allocator.h"
24 #include "api/video/video_bitrate_allocator_factory.h"
25 #include "api/video_codecs/video_encoder.h"
26 #include "api/video_codecs/vp8_temporal_layers.h"
27 #include "api/video_codecs/vp8_temporal_layers_factory.h"
28 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
29 #include "rtc_base/checks.h"
30 #include "rtc_base/ref_counted_object.h"
31 #include "test/gmock.h"
32 #include "test/gtest.h"
33
34 namespace webrtc {
35
36 namespace {
37 static const int kDefaultWidth = 1280;
38 static const int kDefaultHeight = 720;
39 static const int kDefaultFrameRate = 30;
40 static const uint32_t kDefaultMinBitrateBps = 60000;
41 static const uint32_t kDefaultTargetBitrateBps = 2000000;
42 static const uint32_t kDefaultMaxBitrateBps = 2000000;
43 static const uint32_t kDefaultMinTransmitBitrateBps = 400000;
44 static const int kDefaultMaxQp = 48;
45 static const uint32_t kScreenshareTl0BitrateBps = 200000;
46 static const uint32_t kScreenshareCodecTargetBitrateBps = 200000;
47 static const uint32_t kScreenshareDefaultFramerate = 5;
48 // Bitrates for the temporal layers of the higher screenshare simulcast stream.
49 static const uint32_t kHighScreenshareTl0Bps = 800000;
50 static const uint32_t kHighScreenshareTl1Bps = 1200000;
51 } // namespace
52
53 // TODO(sprang): Extend coverage to handle the rest of the codec initializer.
54 class VideoCodecInitializerTest : public ::testing::Test {
55 public:
VideoCodecInitializerTest()56 VideoCodecInitializerTest() {}
~VideoCodecInitializerTest()57 virtual ~VideoCodecInitializerTest() {}
58
59 protected:
SetUpFor(VideoCodecType type,int num_spatial_streams,int num_temporal_streams,bool screenshare)60 void SetUpFor(VideoCodecType type,
61 int num_spatial_streams,
62 int num_temporal_streams,
63 bool screenshare) {
64 config_ = VideoEncoderConfig();
65 config_.codec_type = type;
66
67 if (screenshare) {
68 config_.min_transmit_bitrate_bps = kDefaultMinTransmitBitrateBps;
69 config_.content_type = VideoEncoderConfig::ContentType::kScreen;
70 }
71
72 if (type == VideoCodecType::kVideoCodecVP8) {
73 config_.number_of_streams = num_spatial_streams;
74 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
75 vp8_settings.numberOfTemporalLayers = num_temporal_streams;
76 config_.encoder_specific_settings = new rtc::RefCountedObject<
77 webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
78 } else if (type == VideoCodecType::kVideoCodecVP9) {
79 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
80 vp9_settings.numberOfSpatialLayers = num_spatial_streams;
81 vp9_settings.numberOfTemporalLayers = num_temporal_streams;
82 config_.encoder_specific_settings = new rtc::RefCountedObject<
83 webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
84 } else if (type != VideoCodecType::kVideoCodecMultiplex) {
85 ADD_FAILURE() << "Unexpected codec type: " << type;
86 }
87 }
88
InitializeCodec()89 bool InitializeCodec() {
90 codec_out_ = VideoCodec();
91 frame_buffer_controller_.reset();
92 if (!VideoCodecInitializer::SetupCodec(config_, streams_, &codec_out_)) {
93 return false;
94 }
95 bitrate_allocator_ = CreateBuiltinVideoBitrateAllocatorFactory()
96 ->CreateVideoBitrateAllocator(codec_out_);
97 RTC_CHECK(bitrate_allocator_);
98 if (codec_out_.codecType == VideoCodecType::kVideoCodecMultiplex)
99 return true;
100
101 // Make sure temporal layers instances have been created.
102 if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) {
103 Vp8TemporalLayersFactory factory;
104 const VideoEncoder::Settings settings(VideoEncoder::Capabilities(false),
105 1, 1000);
106 frame_buffer_controller_ =
107 factory.Create(codec_out_, settings, &fec_controller_override_);
108 }
109 return true;
110 }
111
DefaultStream()112 VideoStream DefaultStream() {
113 VideoStream stream;
114 stream.width = kDefaultWidth;
115 stream.height = kDefaultHeight;
116 stream.max_framerate = kDefaultFrameRate;
117 stream.min_bitrate_bps = kDefaultMinBitrateBps;
118 stream.target_bitrate_bps = kDefaultTargetBitrateBps;
119 stream.max_bitrate_bps = kDefaultMaxBitrateBps;
120 stream.max_qp = kDefaultMaxQp;
121 stream.num_temporal_layers = 1;
122 stream.active = true;
123 return stream;
124 }
125
DefaultScreenshareStream()126 VideoStream DefaultScreenshareStream() {
127 VideoStream stream = DefaultStream();
128 stream.min_bitrate_bps = 30000;
129 stream.target_bitrate_bps = kScreenshareTl0BitrateBps;
130 stream.max_bitrate_bps = 1000000;
131 stream.max_framerate = kScreenshareDefaultFramerate;
132 stream.num_temporal_layers = 2;
133 stream.active = true;
134 return stream;
135 }
136
137 MockFecControllerOverride fec_controller_override_;
138
139 // Input settings.
140 VideoEncoderConfig config_;
141 std::vector<VideoStream> streams_;
142
143 // Output.
144 VideoCodec codec_out_;
145 std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_;
146 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_;
147 };
148
TEST_F(VideoCodecInitializerTest,SingleStreamVp8Screenshare)149 TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) {
150 SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true);
151 streams_.push_back(DefaultStream());
152 EXPECT_TRUE(InitializeCodec());
153
154 VideoBitrateAllocation bitrate_allocation =
155 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
156 kDefaultTargetBitrateBps, kDefaultFrameRate));
157 EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
158 EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
159 EXPECT_EQ(kDefaultTargetBitrateBps, bitrate_allocation.get_sum_bps());
160 }
161
TEST_F(VideoCodecInitializerTest,SingleStreamVp8ScreenshareInactive)162 TEST_F(VideoCodecInitializerTest, SingleStreamVp8ScreenshareInactive) {
163 SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true);
164 VideoStream inactive_stream = DefaultStream();
165 inactive_stream.active = false;
166 streams_.push_back(inactive_stream);
167 EXPECT_TRUE(InitializeCodec());
168
169 VideoBitrateAllocation bitrate_allocation =
170 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
171 kDefaultTargetBitrateBps, kDefaultFrameRate));
172 EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
173 EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
174 EXPECT_EQ(0U, bitrate_allocation.get_sum_bps());
175 }
176
TEST_F(VideoCodecInitializerTest,TemporalLayeredVp8Screenshare)177 TEST_F(VideoCodecInitializerTest, TemporalLayeredVp8Screenshare) {
178 SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 2, true);
179 streams_.push_back(DefaultScreenshareStream());
180 EXPECT_TRUE(InitializeCodec());
181
182 EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
183 EXPECT_EQ(2u, codec_out_.VP8()->numberOfTemporalLayers);
184 VideoBitrateAllocation bitrate_allocation =
185 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
186 kScreenshareCodecTargetBitrateBps, kScreenshareDefaultFramerate));
187 EXPECT_EQ(kScreenshareCodecTargetBitrateBps,
188 bitrate_allocation.get_sum_bps());
189 EXPECT_EQ(kScreenshareTl0BitrateBps, bitrate_allocation.GetBitrate(0, 0));
190 }
191
TEST_F(VideoCodecInitializerTest,SimulcastVp8Screenshare)192 TEST_F(VideoCodecInitializerTest, SimulcastVp8Screenshare) {
193 SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true);
194 streams_.push_back(DefaultScreenshareStream());
195 VideoStream video_stream = DefaultStream();
196 video_stream.max_framerate = kScreenshareDefaultFramerate;
197 streams_.push_back(video_stream);
198 EXPECT_TRUE(InitializeCodec());
199
200 EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
201 EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
202 const uint32_t max_bitrate_bps =
203 streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
204 VideoBitrateAllocation bitrate_allocation =
205 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
206 max_bitrate_bps, kScreenshareDefaultFramerate));
207 EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
208 EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
209 bitrate_allocation.GetSpatialLayerSum(0));
210 EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
211 bitrate_allocation.GetSpatialLayerSum(1));
212 }
213
214 // Tests that when a video stream is inactive, then the bitrate allocation will
215 // be 0 for that stream.
TEST_F(VideoCodecInitializerTest,SimulcastVp8ScreenshareInactive)216 TEST_F(VideoCodecInitializerTest, SimulcastVp8ScreenshareInactive) {
217 SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true);
218 streams_.push_back(DefaultScreenshareStream());
219 VideoStream inactive_video_stream = DefaultStream();
220 inactive_video_stream.active = false;
221 inactive_video_stream.max_framerate = kScreenshareDefaultFramerate;
222 streams_.push_back(inactive_video_stream);
223 EXPECT_TRUE(InitializeCodec());
224
225 EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
226 EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
227 const uint32_t target_bitrate =
228 streams_[0].target_bitrate_bps + streams_[1].target_bitrate_bps;
229 VideoBitrateAllocation bitrate_allocation =
230 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
231 target_bitrate, kScreenshareDefaultFramerate));
232 EXPECT_EQ(static_cast<uint32_t>(streams_[0].max_bitrate_bps),
233 bitrate_allocation.get_sum_bps());
234 EXPECT_EQ(static_cast<uint32_t>(streams_[0].max_bitrate_bps),
235 bitrate_allocation.GetSpatialLayerSum(0));
236 EXPECT_EQ(0U, bitrate_allocation.GetSpatialLayerSum(1));
237 }
238
TEST_F(VideoCodecInitializerTest,HighFpsSimulcastVp8Screenshare)239 TEST_F(VideoCodecInitializerTest, HighFpsSimulcastVp8Screenshare) {
240 // Two simulcast streams, the lower one using legacy settings (two temporal
241 // streams, 5fps), the higher one using 3 temporal streams and 30fps.
242 SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 3, true);
243 streams_.push_back(DefaultScreenshareStream());
244 VideoStream video_stream = DefaultStream();
245 video_stream.num_temporal_layers = 3;
246 streams_.push_back(video_stream);
247 EXPECT_TRUE(InitializeCodec());
248
249 EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
250 EXPECT_EQ(3u, codec_out_.VP8()->numberOfTemporalLayers);
251 const uint32_t max_bitrate_bps =
252 streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
253 VideoBitrateAllocation bitrate_allocation = bitrate_allocator_->Allocate(
254 VideoBitrateAllocationParameters(max_bitrate_bps, kDefaultFrameRate));
255 EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
256 EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
257 bitrate_allocation.GetSpatialLayerSum(0));
258 EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
259 bitrate_allocation.GetSpatialLayerSum(1));
260 EXPECT_EQ(kHighScreenshareTl0Bps, bitrate_allocation.GetBitrate(1, 0));
261 EXPECT_EQ(kHighScreenshareTl1Bps - kHighScreenshareTl0Bps,
262 bitrate_allocation.GetBitrate(1, 1));
263 }
264
TEST_F(VideoCodecInitializerTest,SingleStreamMultiplexCodec)265 TEST_F(VideoCodecInitializerTest, SingleStreamMultiplexCodec) {
266 SetUpFor(VideoCodecType::kVideoCodecMultiplex, 1, 1, true);
267 streams_.push_back(DefaultStream());
268 EXPECT_TRUE(InitializeCodec());
269 }
270
TEST_F(VideoCodecInitializerTest,Vp9SvcDefaultLayering)271 TEST_F(VideoCodecInitializerTest, Vp9SvcDefaultLayering) {
272 SetUpFor(VideoCodecType::kVideoCodecVP9, 3, 3, false);
273 VideoStream stream = DefaultStream();
274 stream.num_temporal_layers = 3;
275 streams_.push_back(stream);
276
277 EXPECT_TRUE(InitializeCodec());
278 EXPECT_EQ(codec_out_.VP9()->numberOfSpatialLayers, 3u);
279 EXPECT_EQ(codec_out_.VP9()->numberOfTemporalLayers, 3u);
280 }
281
TEST_F(VideoCodecInitializerTest,Vp9SvcAdjustedLayering)282 TEST_F(VideoCodecInitializerTest, Vp9SvcAdjustedLayering) {
283 SetUpFor(VideoCodecType::kVideoCodecVP9, 3, 3, false);
284 VideoStream stream = DefaultStream();
285 stream.num_temporal_layers = 3;
286 // Set resolution which is only enough to produce 2 spatial layers.
287 stream.width = kMinVp9SpatialLayerWidth * 2;
288 stream.height = kMinVp9SpatialLayerHeight * 2;
289
290 streams_.push_back(stream);
291
292 EXPECT_TRUE(InitializeCodec());
293 EXPECT_EQ(codec_out_.VP9()->numberOfSpatialLayers, 2u);
294 }
295
TEST_F(VideoCodecInitializerTest,Vp9SingleSpatialLayerMaxBitrateIsEqualToCodecMaxBitrate)296 TEST_F(VideoCodecInitializerTest,
297 Vp9SingleSpatialLayerMaxBitrateIsEqualToCodecMaxBitrate) {
298 SetUpFor(VideoCodecType::kVideoCodecVP9, 1, 3, false);
299 VideoStream stream = DefaultStream();
300 stream.num_temporal_layers = 3;
301 streams_.push_back(stream);
302
303 EXPECT_TRUE(InitializeCodec());
304 EXPECT_EQ(codec_out_.spatialLayers[0].maxBitrate,
305 kDefaultMaxBitrateBps / 1000);
306 }
307
TEST_F(VideoCodecInitializerTest,Vp9SingleSpatialLayerTargetBitrateIsEqualToCodecMaxBitrate)308 TEST_F(VideoCodecInitializerTest,
309 Vp9SingleSpatialLayerTargetBitrateIsEqualToCodecMaxBitrate) {
310 SetUpFor(VideoCodecType::kVideoCodecVP9, 1, 1, true);
311 VideoStream stream = DefaultStream();
312 stream.num_temporal_layers = 1;
313 streams_.push_back(stream);
314
315 EXPECT_TRUE(InitializeCodec());
316 EXPECT_EQ(codec_out_.spatialLayers[0].targetBitrate,
317 kDefaultMaxBitrateBps / 1000);
318 }
319
TEST_F(VideoCodecInitializerTest,Vp9KeepBitrateLimitsIfNumberOfSpatialLayersIsReducedToOne)320 TEST_F(VideoCodecInitializerTest,
321 Vp9KeepBitrateLimitsIfNumberOfSpatialLayersIsReducedToOne) {
322 // Request 3 spatial layers for 320x180 input. Actual number of layers will be
323 // reduced to 1 due to low input resolution but SVC bitrate limits should be
324 // applied.
325 SetUpFor(VideoCodecType::kVideoCodecVP9, 3, 3, false);
326 VideoStream stream = DefaultStream();
327 stream.width = 320;
328 stream.height = 180;
329 stream.num_temporal_layers = 3;
330 streams_.push_back(stream);
331
332 EXPECT_TRUE(InitializeCodec());
333 EXPECT_LT(codec_out_.spatialLayers[0].maxBitrate,
334 kDefaultMaxBitrateBps / 1000);
335 }
336
TEST_F(VideoCodecInitializerTest,Vp9DeactivateLayers)337 TEST_F(VideoCodecInitializerTest, Vp9DeactivateLayers) {
338 SetUpFor(VideoCodecType::kVideoCodecVP9, 3, 1, false);
339 VideoStream stream = DefaultStream();
340 streams_.push_back(stream);
341
342 config_.simulcast_layers.resize(3);
343
344 // Activate all layers.
345 config_.simulcast_layers[0].active = true;
346 config_.simulcast_layers[1].active = true;
347 config_.simulcast_layers[2].active = true;
348 EXPECT_TRUE(InitializeCodec());
349 EXPECT_TRUE(codec_out_.spatialLayers[0].active);
350 EXPECT_TRUE(codec_out_.spatialLayers[1].active);
351 EXPECT_TRUE(codec_out_.spatialLayers[2].active);
352
353 // Deactivate top layer.
354 config_.simulcast_layers[2].active = false;
355 EXPECT_TRUE(InitializeCodec());
356 EXPECT_TRUE(codec_out_.spatialLayers[0].active);
357 EXPECT_TRUE(codec_out_.spatialLayers[1].active);
358 EXPECT_FALSE(codec_out_.spatialLayers[2].active);
359
360 // Deactivate middle layer.
361 config_.simulcast_layers[2].active = true;
362 config_.simulcast_layers[1].active = false;
363 EXPECT_TRUE(InitializeCodec());
364 EXPECT_TRUE(codec_out_.spatialLayers[0].active);
365 EXPECT_FALSE(codec_out_.spatialLayers[1].active);
366 EXPECT_TRUE(codec_out_.spatialLayers[2].active);
367 }
368
369 } // namespace webrtc
370