1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <memory>
12
13 #include "api/test/simulated_network.h"
14 #include "api/test/video/function_video_encoder_factory.h"
15 #include "call/fake_network_pipe.h"
16 #include "call/simulated_network.h"
17 #include "media/engine/internal_encoder_factory.h"
18 #include "media/engine/simulcast_encoder_adapter.h"
19 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
20 #include "modules/rtp_rtcp/source/rtp_packet.h"
21 #include "modules/video_coding/codecs/vp8/include/vp8.h"
22 #include "modules/video_coding/codecs/vp9/include/vp9.h"
23 #include "rtc_base/numerics/safe_conversions.h"
24 #include "rtc_base/numerics/sequence_number_util.h"
25 #include "rtc_base/synchronization/mutex.h"
26 #include "rtc_base/task_queue_for_test.h"
27 #include "test/call_test.h"
28
29 namespace webrtc {
30 namespace {
31 const int kFrameMaxWidth = 1280;
32 const int kFrameMaxHeight = 720;
33 const int kFrameRate = 30;
34 const int kMaxSecondsLost = 5;
35 const int kMaxFramesLost = kFrameRate * kMaxSecondsLost;
36 const int kMinPacketsToObserve = 10;
37 const int kEncoderBitrateBps = 300000;
38 const uint32_t kPictureIdWraparound = (1 << 15);
39 const size_t kNumTemporalLayers[] = {1, 2, 3};
40
41 } // namespace
42
43 class PictureIdObserver : public test::RtpRtcpObserver {
44 public:
PictureIdObserver(VideoCodecType codec_type)45 explicit PictureIdObserver(VideoCodecType codec_type)
46 : test::RtpRtcpObserver(test::CallTest::kDefaultTimeoutMs),
47 depacketizer_(CreateVideoRtpDepacketizer(codec_type)),
48 max_expected_picture_id_gap_(0),
49 max_expected_tl0_idx_gap_(0),
50 num_ssrcs_to_observe_(1) {}
51
SetExpectedSsrcs(size_t num_expected_ssrcs)52 void SetExpectedSsrcs(size_t num_expected_ssrcs) {
53 MutexLock lock(&mutex_);
54 num_ssrcs_to_observe_ = num_expected_ssrcs;
55 }
56
ResetObservedSsrcs()57 void ResetObservedSsrcs() {
58 MutexLock lock(&mutex_);
59 // Do not clear the timestamp and picture_id, to ensure that we check
60 // consistency between reinits and recreations.
61 num_packets_sent_.clear();
62 observed_ssrcs_.clear();
63 }
64
SetMaxExpectedPictureIdGap(int max_expected_picture_id_gap)65 void SetMaxExpectedPictureIdGap(int max_expected_picture_id_gap) {
66 MutexLock lock(&mutex_);
67 max_expected_picture_id_gap_ = max_expected_picture_id_gap;
68 // Expect smaller gap for |tl0_pic_idx| (running index for temporal_idx 0).
69 max_expected_tl0_idx_gap_ = max_expected_picture_id_gap_ / 2;
70 }
71
72 private:
73 struct ParsedPacket {
74 uint32_t timestamp;
75 uint32_t ssrc;
76 int16_t picture_id;
77 int16_t tl0_pic_idx;
78 uint8_t temporal_idx;
79 VideoFrameType frame_type;
80 };
81
ParsePayload(const uint8_t * packet,size_t length,ParsedPacket * parsed) const82 bool ParsePayload(const uint8_t* packet,
83 size_t length,
84 ParsedPacket* parsed) const {
85 RtpPacket rtp_packet;
86 EXPECT_TRUE(rtp_packet.Parse(packet, length));
87 EXPECT_TRUE(rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[0] ||
88 rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[1] ||
89 rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[2])
90 << "Unknown SSRC sent.";
91
92 if (rtp_packet.payload_size() == 0) {
93 return false; // Padding packet.
94 }
95
96 parsed->timestamp = rtp_packet.Timestamp();
97 parsed->ssrc = rtp_packet.Ssrc();
98
99 absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
100 depacketizer_->Parse(rtp_packet.PayloadBuffer());
101 EXPECT_TRUE(parsed_payload);
102
103 if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
104 &parsed_payload->video_header.video_type_header)) {
105 parsed->picture_id = vp8_header->pictureId;
106 parsed->tl0_pic_idx = vp8_header->tl0PicIdx;
107 parsed->temporal_idx = vp8_header->temporalIdx;
108 } else if (const auto* vp9_header = absl::get_if<RTPVideoHeaderVP9>(
109 &parsed_payload->video_header.video_type_header)) {
110 parsed->picture_id = vp9_header->picture_id;
111 parsed->tl0_pic_idx = vp9_header->tl0_pic_idx;
112 parsed->temporal_idx = vp9_header->temporal_idx;
113 } else {
114 RTC_NOTREACHED();
115 }
116
117 parsed->frame_type = parsed_payload->video_header.frame_type;
118 return true;
119 }
120
121 // Verify continuity and monotonicity of picture_id sequence.
VerifyPictureId(const ParsedPacket & current,const ParsedPacket & last) const122 void VerifyPictureId(const ParsedPacket& current,
123 const ParsedPacket& last) const
124 RTC_EXCLUSIVE_LOCKS_REQUIRED(&mutex_) {
125 if (current.timestamp == last.timestamp) {
126 EXPECT_EQ(last.picture_id, current.picture_id);
127 return; // Same frame.
128 }
129
130 // Packet belongs to a new frame.
131 // Picture id should be increasing.
132 EXPECT_TRUE((AheadOf<uint16_t, kPictureIdWraparound>(current.picture_id,
133 last.picture_id)));
134
135 // Expect continuously increasing picture id.
136 int diff = ForwardDiff<uint16_t, kPictureIdWraparound>(last.picture_id,
137 current.picture_id);
138 if (diff > 1) {
139 // If the VideoSendStream is destroyed, any frames still in queue is lost.
140 // Gaps only possible for first frame after a recreation, i.e. key frames.
141 EXPECT_EQ(VideoFrameType::kVideoFrameKey, current.frame_type);
142 EXPECT_LE(diff - 1, max_expected_picture_id_gap_);
143 }
144 }
145
VerifyTl0Idx(const ParsedPacket & current,const ParsedPacket & last) const146 void VerifyTl0Idx(const ParsedPacket& current, const ParsedPacket& last) const
147 RTC_EXCLUSIVE_LOCKS_REQUIRED(&mutex_) {
148 if (current.tl0_pic_idx == kNoTl0PicIdx ||
149 current.temporal_idx == kNoTemporalIdx) {
150 return; // No temporal layers.
151 }
152
153 if (current.timestamp == last.timestamp || current.temporal_idx != 0) {
154 EXPECT_EQ(last.tl0_pic_idx, current.tl0_pic_idx);
155 return;
156 }
157
158 // New frame with |temporal_idx| 0.
159 // |tl0_pic_idx| should be increasing.
160 EXPECT_TRUE(AheadOf<uint8_t>(current.tl0_pic_idx, last.tl0_pic_idx));
161
162 // Expect continuously increasing idx.
163 int diff = ForwardDiff<uint8_t>(last.tl0_pic_idx, current.tl0_pic_idx);
164 if (diff > 1) {
165 // If the VideoSendStream is destroyed, any frames still in queue is lost.
166 // Gaps only possible for first frame after a recreation, i.e. key frames.
167 EXPECT_EQ(VideoFrameType::kVideoFrameKey, current.frame_type);
168 EXPECT_LE(diff - 1, max_expected_tl0_idx_gap_);
169 }
170 }
171
OnSendRtp(const uint8_t * packet,size_t length)172 Action OnSendRtp(const uint8_t* packet, size_t length) override {
173 MutexLock lock(&mutex_);
174
175 ParsedPacket parsed;
176 if (!ParsePayload(packet, length, &parsed))
177 return SEND_PACKET;
178
179 uint32_t ssrc = parsed.ssrc;
180 if (last_observed_packet_.find(ssrc) != last_observed_packet_.end()) {
181 // Compare to last packet.
182 VerifyPictureId(parsed, last_observed_packet_[ssrc]);
183 VerifyTl0Idx(parsed, last_observed_packet_[ssrc]);
184 }
185
186 last_observed_packet_[ssrc] = parsed;
187
188 // Pass the test when enough media packets have been received on all
189 // streams.
190 if (++num_packets_sent_[ssrc] >= kMinPacketsToObserve &&
191 observed_ssrcs_.find(ssrc) == observed_ssrcs_.end()) {
192 observed_ssrcs_.insert(ssrc);
193 if (observed_ssrcs_.size() == num_ssrcs_to_observe_) {
194 observation_complete_.Set();
195 }
196 }
197 return SEND_PACKET;
198 }
199
200 Mutex mutex_;
201 const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
202 std::map<uint32_t, ParsedPacket> last_observed_packet_ RTC_GUARDED_BY(mutex_);
203 std::map<uint32_t, size_t> num_packets_sent_ RTC_GUARDED_BY(mutex_);
204 int max_expected_picture_id_gap_ RTC_GUARDED_BY(mutex_);
205 int max_expected_tl0_idx_gap_ RTC_GUARDED_BY(mutex_);
206 size_t num_ssrcs_to_observe_ RTC_GUARDED_BY(mutex_);
207 std::set<uint32_t> observed_ssrcs_ RTC_GUARDED_BY(mutex_);
208 };
209
210 class PictureIdTest : public test::CallTest,
211 public ::testing::WithParamInterface<size_t> {
212 public:
PictureIdTest()213 PictureIdTest() : num_temporal_layers_(GetParam()) {}
214
~PictureIdTest()215 virtual ~PictureIdTest() {
216 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
217 send_transport_.reset();
218 receive_transport_.reset();
219 DestroyCalls();
220 });
221 }
222
223 void SetupEncoder(VideoEncoderFactory* encoder_factory,
224 const std::string& payload_name);
225 void SetVideoEncoderConfig(int num_streams);
226 void TestPictureIdContinuousAfterReconfigure(
227 const std::vector<int>& ssrc_counts);
228 void TestPictureIdIncreaseAfterRecreateStreams(
229 const std::vector<int>& ssrc_counts);
230
231 private:
232 const size_t num_temporal_layers_;
233 std::unique_ptr<PictureIdObserver> observer_;
234 };
235
236 INSTANTIATE_TEST_SUITE_P(TemporalLayers,
237 PictureIdTest,
238 ::testing::ValuesIn(kNumTemporalLayers));
239
SetupEncoder(VideoEncoderFactory * encoder_factory,const std::string & payload_name)240 void PictureIdTest::SetupEncoder(VideoEncoderFactory* encoder_factory,
241 const std::string& payload_name) {
242 observer_.reset(
243 new PictureIdObserver(PayloadStringToCodecType(payload_name)));
244
245 SendTask(
246 RTC_FROM_HERE, task_queue(), [this, encoder_factory, payload_name]() {
247 CreateCalls();
248
249 send_transport_.reset(new test::PacketTransport(
250 task_queue(), sender_call_.get(), observer_.get(),
251 test::PacketTransport::kSender, payload_type_map_,
252 std::make_unique<FakeNetworkPipe>(
253 Clock::GetRealTimeClock(),
254 std::make_unique<SimulatedNetwork>(
255 BuiltInNetworkBehaviorConfig()))));
256
257 CreateSendConfig(kNumSimulcastStreams, 0, 0, send_transport_.get());
258 GetVideoSendConfig()->encoder_settings.encoder_factory =
259 encoder_factory;
260 GetVideoSendConfig()->rtp.payload_name = payload_name;
261 GetVideoEncoderConfig()->codec_type =
262 PayloadStringToCodecType(payload_name);
263 SetVideoEncoderConfig(/* number_of_streams */ 1);
264 });
265 }
266
SetVideoEncoderConfig(int num_streams)267 void PictureIdTest::SetVideoEncoderConfig(int num_streams) {
268 GetVideoEncoderConfig()->number_of_streams = num_streams;
269 GetVideoEncoderConfig()->max_bitrate_bps = kEncoderBitrateBps;
270
271 // Always divide the same total bitrate across all streams so that sending a
272 // single stream avoids lowering the bitrate estimate and requiring a
273 // subsequent rampup.
274 const int encoder_stream_bps = kEncoderBitrateBps / num_streams;
275 double scale_factor = 1.0;
276 for (int i = num_streams - 1; i >= 0; --i) {
277 VideoStream& stream = GetVideoEncoderConfig()->simulcast_layers[i];
278 // Reduce the min bitrate by 10% to account for overhead that might
279 // otherwise cause streams to not be enabled.
280 stream.min_bitrate_bps = static_cast<int>(encoder_stream_bps * 0.9);
281 stream.target_bitrate_bps = encoder_stream_bps;
282 stream.max_bitrate_bps = encoder_stream_bps;
283 stream.num_temporal_layers = num_temporal_layers_;
284 stream.scale_resolution_down_by = scale_factor;
285 scale_factor *= 2.0;
286 }
287 }
288
TestPictureIdContinuousAfterReconfigure(const std::vector<int> & ssrc_counts)289 void PictureIdTest::TestPictureIdContinuousAfterReconfigure(
290 const std::vector<int>& ssrc_counts) {
291 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
292 CreateVideoStreams();
293 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
294
295 // Initial test with a single stream.
296 Start();
297 });
298
299 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
300
301 // Reconfigure VideoEncoder and test picture id increase.
302 // Expect continuously increasing picture id, equivalent to no gaps.
303 observer_->SetMaxExpectedPictureIdGap(0);
304 for (int ssrc_count : ssrc_counts) {
305 SetVideoEncoderConfig(ssrc_count);
306 observer_->SetExpectedSsrcs(ssrc_count);
307 observer_->ResetObservedSsrcs();
308 // Make sure the picture_id sequence is continuous on reinit and recreate.
309 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
310 GetVideoSendStream()->ReconfigureVideoEncoder(
311 GetVideoEncoderConfig()->Copy());
312 });
313 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
314 }
315
316 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
317 Stop();
318 DestroyStreams();
319 });
320 }
321
TestPictureIdIncreaseAfterRecreateStreams(const std::vector<int> & ssrc_counts)322 void PictureIdTest::TestPictureIdIncreaseAfterRecreateStreams(
323 const std::vector<int>& ssrc_counts) {
324 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
325 CreateVideoStreams();
326 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
327
328 // Initial test with a single stream.
329 Start();
330 });
331
332 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
333
334 // Recreate VideoSendStream and test picture id increase.
335 // When the VideoSendStream is destroyed, any frames still in queue is lost
336 // with it, therefore it is expected that some frames might be lost.
337 observer_->SetMaxExpectedPictureIdGap(kMaxFramesLost);
338 for (int ssrc_count : ssrc_counts) {
339 SendTask(RTC_FROM_HERE, task_queue(), [this, &ssrc_count]() {
340 DestroyVideoSendStreams();
341
342 SetVideoEncoderConfig(ssrc_count);
343 observer_->SetExpectedSsrcs(ssrc_count);
344 observer_->ResetObservedSsrcs();
345
346 CreateVideoSendStreams();
347 GetVideoSendStream()->Start();
348 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
349 });
350
351 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
352 }
353
354 SendTask(RTC_FROM_HERE, task_queue(), [this]() {
355 Stop();
356 DestroyStreams();
357 });
358 }
359
TEST_P(PictureIdTest,ContinuousAfterReconfigureVp8)360 TEST_P(PictureIdTest, ContinuousAfterReconfigureVp8) {
361 test::FunctionVideoEncoderFactory encoder_factory(
362 []() { return VP8Encoder::Create(); });
363 SetupEncoder(&encoder_factory, "VP8");
364 TestPictureIdContinuousAfterReconfigure({1, 3, 3, 1, 1});
365 }
366
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamVp8)367 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamVp8) {
368 test::FunctionVideoEncoderFactory encoder_factory(
369 []() { return VP8Encoder::Create(); });
370 SetupEncoder(&encoder_factory, "VP8");
371 TestPictureIdIncreaseAfterRecreateStreams({1, 3, 3, 1, 1});
372 }
373
TEST_P(PictureIdTest,ContinuousAfterStreamCountChangeVp8)374 TEST_P(PictureIdTest, ContinuousAfterStreamCountChangeVp8) {
375 test::FunctionVideoEncoderFactory encoder_factory(
376 []() { return VP8Encoder::Create(); });
377 // Make sure that the picture id is not reset if the stream count goes
378 // down and then up.
379 SetupEncoder(&encoder_factory, "VP8");
380 TestPictureIdContinuousAfterReconfigure({3, 1, 3});
381 }
382
TEST_P(PictureIdTest,ContinuousAfterReconfigureSimulcastEncoderAdapter)383 TEST_P(PictureIdTest, ContinuousAfterReconfigureSimulcastEncoderAdapter) {
384 InternalEncoderFactory internal_encoder_factory;
385 test::FunctionVideoEncoderFactory encoder_factory(
386 [&internal_encoder_factory]() {
387 return std::make_unique<SimulcastEncoderAdapter>(
388 &internal_encoder_factory, SdpVideoFormat("VP8"));
389 });
390 SetupEncoder(&encoder_factory, "VP8");
391 TestPictureIdContinuousAfterReconfigure({1, 3, 3, 1, 1});
392 }
393
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamSimulcastEncoderAdapter)394 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamSimulcastEncoderAdapter) {
395 InternalEncoderFactory internal_encoder_factory;
396 test::FunctionVideoEncoderFactory encoder_factory(
397 [&internal_encoder_factory]() {
398 return std::make_unique<SimulcastEncoderAdapter>(
399 &internal_encoder_factory, SdpVideoFormat("VP8"));
400 });
401 SetupEncoder(&encoder_factory, "VP8");
402 TestPictureIdIncreaseAfterRecreateStreams({1, 3, 3, 1, 1});
403 }
404
TEST_P(PictureIdTest,ContinuousAfterStreamCountChangeSimulcastEncoderAdapter)405 TEST_P(PictureIdTest, ContinuousAfterStreamCountChangeSimulcastEncoderAdapter) {
406 InternalEncoderFactory internal_encoder_factory;
407 test::FunctionVideoEncoderFactory encoder_factory(
408 [&internal_encoder_factory]() {
409 return std::make_unique<SimulcastEncoderAdapter>(
410 &internal_encoder_factory, SdpVideoFormat("VP8"));
411 });
412 // Make sure that the picture id is not reset if the stream count goes
413 // down and then up.
414 SetupEncoder(&encoder_factory, "VP8");
415 TestPictureIdContinuousAfterReconfigure({3, 1, 3});
416 }
417
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamVp9)418 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamVp9) {
419 test::FunctionVideoEncoderFactory encoder_factory(
420 []() { return VP9Encoder::Create(); });
421 SetupEncoder(&encoder_factory, "VP9");
422 TestPictureIdIncreaseAfterRecreateStreams({1, 1});
423 }
424
425 } // namespace webrtc
426