1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <memory>
12
13 #include "api/task_queue/task_queue_base.h"
14 #include "api/test/simulated_network.h"
15 #include "api/test/video/function_video_encoder_factory.h"
16 #include "call/fake_network_pipe.h"
17 #include "call/simulated_network.h"
18 #include "media/engine/internal_decoder_factory.h"
19 #include "modules/include/module_common_types_public.h"
20 #include "modules/rtp_rtcp/source/byte_io.h"
21 #include "modules/rtp_rtcp/source/rtp_packet.h"
22 #include "modules/video_coding/codecs/vp8/include/vp8.h"
23 #include "rtc_base/synchronization/mutex.h"
24 #include "test/call_test.h"
25 #include "test/field_trial.h"
26 #include "test/gmock.h"
27 #include "test/gtest.h"
28 #include "test/rtcp_packet_parser.h"
29
30 using ::testing::Contains;
31 using ::testing::Not;
32
33 namespace webrtc {
34 namespace {
35 enum : int { // The first valid value is 1.
36 kTransportSequenceNumberExtensionId = 1,
37 kVideoRotationExtensionId,
38 };
39 } // namespace
40
41 class FecEndToEndTest : public test::CallTest {
42 public:
FecEndToEndTest()43 FecEndToEndTest() {
44 RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
45 kTransportSequenceNumberExtensionId));
46 RegisterRtpExtension(RtpExtension(RtpExtension::kVideoRotationUri,
47 kVideoRotationExtensionId));
48 }
49 };
50
TEST_F(FecEndToEndTest,ReceivesUlpfec)51 TEST_F(FecEndToEndTest, ReceivesUlpfec) {
52 class UlpfecRenderObserver : public test::EndToEndTest,
53 public rtc::VideoSinkInterface<VideoFrame> {
54 public:
55 UlpfecRenderObserver()
56 : EndToEndTest(kDefaultTimeoutMs),
57 encoder_factory_([]() { return VP8Encoder::Create(); }),
58 random_(0xcafef00d1),
59 num_packets_sent_(0) {}
60
61 private:
62 Action OnSendRtp(const uint8_t* packet, size_t length) override {
63 MutexLock lock(&mutex_);
64 RtpPacket rtp_packet;
65 EXPECT_TRUE(rtp_packet.Parse(packet, length));
66
67 EXPECT_TRUE(rtp_packet.PayloadType() == kVideoSendPayloadType ||
68 rtp_packet.PayloadType() == kRedPayloadType)
69 << "Unknown payload type received.";
70 EXPECT_EQ(kVideoSendSsrcs[0], rtp_packet.Ssrc())
71 << "Unknown SSRC received.";
72
73 // Parse RED header.
74 int encapsulated_payload_type = -1;
75 if (rtp_packet.PayloadType() == kRedPayloadType) {
76 encapsulated_payload_type = rtp_packet.payload()[0];
77
78 EXPECT_TRUE(encapsulated_payload_type == kVideoSendPayloadType ||
79 encapsulated_payload_type == kUlpfecPayloadType)
80 << "Unknown encapsulated payload type received.";
81 }
82
83 // To minimize test flakiness, always let ULPFEC packets through.
84 if (encapsulated_payload_type == kUlpfecPayloadType) {
85 return SEND_PACKET;
86 }
87
88 // Simulate 5% video packet loss after rampup period. Record the
89 // corresponding timestamps that were dropped.
90 if (num_packets_sent_++ > 100 && random_.Rand(1, 100) <= 5) {
91 if (encapsulated_payload_type == kVideoSendPayloadType) {
92 dropped_sequence_numbers_.insert(rtp_packet.SequenceNumber());
93 dropped_timestamps_.insert(rtp_packet.Timestamp());
94 }
95 return DROP_PACKET;
96 }
97
98 return SEND_PACKET;
99 }
100
101 void OnFrame(const VideoFrame& video_frame) override {
102 MutexLock lock(&mutex_);
103 // Rendering frame with timestamp of packet that was dropped -> FEC
104 // protection worked.
105 auto it = dropped_timestamps_.find(video_frame.timestamp());
106 if (it != dropped_timestamps_.end()) {
107 observation_complete_.Set();
108 }
109 }
110
111 void ModifyVideoConfigs(
112 VideoSendStream::Config* send_config,
113 std::vector<VideoReceiveStream::Config>* receive_configs,
114 VideoEncoderConfig* encoder_config) override {
115 // Use VP8 instead of FAKE, since the latter does not have PictureID
116 // in the packetization headers.
117 send_config->encoder_settings.encoder_factory = &encoder_factory_;
118 send_config->rtp.payload_name = "VP8";
119 send_config->rtp.payload_type = kVideoSendPayloadType;
120 encoder_config->codec_type = kVideoCodecVP8;
121 VideoReceiveStream::Decoder decoder =
122 test::CreateMatchingDecoder(*send_config);
123 decoder.decoder_factory = &decoder_factory_;
124 (*receive_configs)[0].decoders.clear();
125 (*receive_configs)[0].decoders.push_back(decoder);
126
127 // Enable ULPFEC over RED.
128 send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
129 send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
130 (*receive_configs)[0].rtp.red_payload_type = kRedPayloadType;
131 (*receive_configs)[0].rtp.ulpfec_payload_type = kUlpfecPayloadType;
132
133 (*receive_configs)[0].renderer = this;
134 }
135
136 void PerformTest() override {
137 EXPECT_TRUE(Wait())
138 << "Timed out waiting for dropped frames to be rendered.";
139 }
140
141 Mutex mutex_;
142 std::unique_ptr<VideoEncoder> encoder_;
143 test::FunctionVideoEncoderFactory encoder_factory_;
144 InternalDecoderFactory decoder_factory_;
145 std::set<uint32_t> dropped_sequence_numbers_ RTC_GUARDED_BY(mutex_);
146 // Several packets can have the same timestamp.
147 std::multiset<uint32_t> dropped_timestamps_ RTC_GUARDED_BY(mutex_);
148 Random random_;
149 int num_packets_sent_ RTC_GUARDED_BY(mutex_);
150 } test;
151
152 RunBaseTest(&test);
153 }
154
155 class FlexfecRenderObserver : public test::EndToEndTest,
156 public rtc::VideoSinkInterface<VideoFrame> {
157 public:
158 static constexpr uint32_t kVideoLocalSsrc = 123;
159 static constexpr uint32_t kFlexfecLocalSsrc = 456;
160
FlexfecRenderObserver(bool enable_nack,bool expect_flexfec_rtcp)161 explicit FlexfecRenderObserver(bool enable_nack, bool expect_flexfec_rtcp)
162 : test::EndToEndTest(test::CallTest::kDefaultTimeoutMs),
163 enable_nack_(enable_nack),
164 expect_flexfec_rtcp_(expect_flexfec_rtcp),
165 received_flexfec_rtcp_(false),
166 random_(0xcafef00d1),
167 num_packets_sent_(0) {}
168
GetNumFlexfecStreams() const169 size_t GetNumFlexfecStreams() const override { return 1; }
170
171 private:
OnSendRtp(const uint8_t * packet,size_t length)172 Action OnSendRtp(const uint8_t* packet, size_t length) override {
173 MutexLock lock(&mutex_);
174 RtpPacket rtp_packet;
175 EXPECT_TRUE(rtp_packet.Parse(packet, length));
176
177 EXPECT_TRUE(
178 rtp_packet.PayloadType() == test::CallTest::kFakeVideoSendPayloadType ||
179 rtp_packet.PayloadType() == test::CallTest::kFlexfecPayloadType ||
180 (enable_nack_ &&
181 rtp_packet.PayloadType() == test::CallTest::kSendRtxPayloadType))
182 << "Unknown payload type received.";
183 EXPECT_TRUE(
184 rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[0] ||
185 rtp_packet.Ssrc() == test::CallTest::kFlexfecSendSsrc ||
186 (enable_nack_ && rtp_packet.Ssrc() == test::CallTest::kSendRtxSsrcs[0]))
187 << "Unknown SSRC received.";
188
189 // To reduce test flakiness, always let FlexFEC packets through.
190 if (rtp_packet.PayloadType() == test::CallTest::kFlexfecPayloadType) {
191 EXPECT_EQ(test::CallTest::kFlexfecSendSsrc, rtp_packet.Ssrc());
192
193 return SEND_PACKET;
194 }
195
196 // To reduce test flakiness, always let RTX packets through.
197 if (rtp_packet.PayloadType() == test::CallTest::kSendRtxPayloadType) {
198 EXPECT_EQ(test::CallTest::kSendRtxSsrcs[0], rtp_packet.Ssrc());
199
200 if (rtp_packet.payload_size() == 0) {
201 // Pure padding packet.
202 return SEND_PACKET;
203 }
204
205 // Parse RTX header.
206 uint16_t original_sequence_number =
207 ByteReader<uint16_t>::ReadBigEndian(rtp_packet.payload().data());
208
209 // From the perspective of FEC, a retransmitted packet is no longer
210 // dropped, so remove it from list of dropped packets.
211 auto seq_num_it =
212 dropped_sequence_numbers_.find(original_sequence_number);
213 if (seq_num_it != dropped_sequence_numbers_.end()) {
214 dropped_sequence_numbers_.erase(seq_num_it);
215 auto ts_it = dropped_timestamps_.find(rtp_packet.Timestamp());
216 EXPECT_NE(ts_it, dropped_timestamps_.end());
217 dropped_timestamps_.erase(ts_it);
218 }
219
220 return SEND_PACKET;
221 }
222
223 // Simulate 5% video packet loss after rampup period. Record the
224 // corresponding timestamps that were dropped.
225 if (num_packets_sent_++ > 100 && random_.Rand(1, 100) <= 5) {
226 EXPECT_EQ(test::CallTest::kFakeVideoSendPayloadType,
227 rtp_packet.PayloadType());
228 EXPECT_EQ(test::CallTest::kVideoSendSsrcs[0], rtp_packet.Ssrc());
229
230 dropped_sequence_numbers_.insert(rtp_packet.SequenceNumber());
231 dropped_timestamps_.insert(rtp_packet.Timestamp());
232
233 return DROP_PACKET;
234 }
235
236 return SEND_PACKET;
237 }
238
OnReceiveRtcp(const uint8_t * data,size_t length)239 Action OnReceiveRtcp(const uint8_t* data, size_t length) override {
240 test::RtcpPacketParser parser;
241
242 parser.Parse(data, length);
243 if (parser.sender_ssrc() == kFlexfecLocalSsrc) {
244 EXPECT_EQ(1, parser.receiver_report()->num_packets());
245 const std::vector<rtcp::ReportBlock>& report_blocks =
246 parser.receiver_report()->report_blocks();
247 if (!report_blocks.empty()) {
248 EXPECT_EQ(1U, report_blocks.size());
249 EXPECT_EQ(test::CallTest::kFlexfecSendSsrc,
250 report_blocks[0].source_ssrc());
251 MutexLock lock(&mutex_);
252 received_flexfec_rtcp_ = true;
253 }
254 }
255
256 return SEND_PACKET;
257 }
258
CreateSendTransport(TaskQueueBase * task_queue,Call * sender_call)259 std::unique_ptr<test::PacketTransport> CreateSendTransport(
260 TaskQueueBase* task_queue,
261 Call* sender_call) override {
262 // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
263 const int kNetworkDelayMs = 100;
264 BuiltInNetworkBehaviorConfig config;
265 config.queue_delay_ms = kNetworkDelayMs;
266 return std::make_unique<test::PacketTransport>(
267 task_queue, sender_call, this, test::PacketTransport::kSender,
268 test::CallTest::payload_type_map_,
269 std::make_unique<FakeNetworkPipe>(
270 Clock::GetRealTimeClock(),
271 std::make_unique<SimulatedNetwork>(config)));
272 }
273
OnFrame(const VideoFrame & video_frame)274 void OnFrame(const VideoFrame& video_frame) override {
275 EXPECT_EQ(kVideoRotation_90, video_frame.rotation());
276
277 MutexLock lock(&mutex_);
278 // Rendering frame with timestamp of packet that was dropped -> FEC
279 // protection worked.
280 auto it = dropped_timestamps_.find(video_frame.timestamp());
281 if (it != dropped_timestamps_.end()) {
282 if (!expect_flexfec_rtcp_ || received_flexfec_rtcp_) {
283 observation_complete_.Set();
284 }
285 }
286 }
287
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStream::Config> * receive_configs,VideoEncoderConfig * encoder_config)288 void ModifyVideoConfigs(
289 VideoSendStream::Config* send_config,
290 std::vector<VideoReceiveStream::Config>* receive_configs,
291 VideoEncoderConfig* encoder_config) override {
292 (*receive_configs)[0].rtp.local_ssrc = kVideoLocalSsrc;
293 (*receive_configs)[0].renderer = this;
294
295 if (enable_nack_) {
296 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
297 send_config->rtp.rtx.ssrcs.push_back(test::CallTest::kSendRtxSsrcs[0]);
298 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
299
300 (*receive_configs)[0].rtp.nack.rtp_history_ms =
301 test::CallTest::kNackRtpHistoryMs;
302 (*receive_configs)[0].rtp.rtx_ssrc = test::CallTest::kSendRtxSsrcs[0];
303 (*receive_configs)[0]
304 .rtp
305 .rtx_associated_payload_types[test::CallTest::kSendRtxPayloadType] =
306 test::CallTest::kVideoSendPayloadType;
307 }
308 }
309
OnFrameGeneratorCapturerCreated(test::FrameGeneratorCapturer * frame_generator_capturer)310 void OnFrameGeneratorCapturerCreated(
311 test::FrameGeneratorCapturer* frame_generator_capturer) override {
312 frame_generator_capturer->SetFakeRotation(kVideoRotation_90);
313 }
314
ModifyFlexfecConfigs(std::vector<FlexfecReceiveStream::Config> * receive_configs)315 void ModifyFlexfecConfigs(
316 std::vector<FlexfecReceiveStream::Config>* receive_configs) override {
317 (*receive_configs)[0].local_ssrc = kFlexfecLocalSsrc;
318 }
319
PerformTest()320 void PerformTest() override {
321 EXPECT_TRUE(Wait())
322 << "Timed out waiting for dropped frames to be rendered.";
323 }
324
325 Mutex mutex_;
326 std::set<uint32_t> dropped_sequence_numbers_ RTC_GUARDED_BY(mutex_);
327 // Several packets can have the same timestamp.
328 std::multiset<uint32_t> dropped_timestamps_ RTC_GUARDED_BY(mutex_);
329 const bool enable_nack_;
330 const bool expect_flexfec_rtcp_;
331 bool received_flexfec_rtcp_ RTC_GUARDED_BY(mutex_);
332 Random random_;
333 int num_packets_sent_;
334 };
335
TEST_F(FecEndToEndTest,RecoversWithFlexfec)336 TEST_F(FecEndToEndTest, RecoversWithFlexfec) {
337 FlexfecRenderObserver test(false, false);
338 RunBaseTest(&test);
339 }
340
TEST_F(FecEndToEndTest,RecoversWithFlexfecAndNack)341 TEST_F(FecEndToEndTest, RecoversWithFlexfecAndNack) {
342 FlexfecRenderObserver test(true, false);
343 RunBaseTest(&test);
344 }
345
TEST_F(FecEndToEndTest,RecoversWithFlexfecAndSendsCorrespondingRtcp)346 TEST_F(FecEndToEndTest, RecoversWithFlexfecAndSendsCorrespondingRtcp) {
347 FlexfecRenderObserver test(false, true);
348 RunBaseTest(&test);
349 }
350
TEST_F(FecEndToEndTest,ReceivedUlpfecPacketsNotNacked)351 TEST_F(FecEndToEndTest, ReceivedUlpfecPacketsNotNacked) {
352 class UlpfecNackObserver : public test::EndToEndTest {
353 public:
354 UlpfecNackObserver()
355 : EndToEndTest(kDefaultTimeoutMs),
356 state_(kFirstPacket),
357 ulpfec_sequence_number_(0),
358 has_last_sequence_number_(false),
359 last_sequence_number_(0),
360 encoder_factory_([]() { return VP8Encoder::Create(); }) {}
361
362 private:
363 Action OnSendRtp(const uint8_t* packet, size_t length) override {
364 MutexLock lock_(&mutex_);
365 RtpPacket rtp_packet;
366 EXPECT_TRUE(rtp_packet.Parse(packet, length));
367
368 int encapsulated_payload_type = -1;
369 if (rtp_packet.PayloadType() == kRedPayloadType) {
370 encapsulated_payload_type = rtp_packet.payload()[0];
371 if (encapsulated_payload_type != kFakeVideoSendPayloadType)
372 EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type);
373 } else {
374 EXPECT_EQ(kFakeVideoSendPayloadType, rtp_packet.PayloadType());
375 }
376
377 if (has_last_sequence_number_ &&
378 !IsNewerSequenceNumber(rtp_packet.SequenceNumber(),
379 last_sequence_number_)) {
380 // Drop retransmitted packets.
381 return DROP_PACKET;
382 }
383 last_sequence_number_ = rtp_packet.SequenceNumber();
384 has_last_sequence_number_ = true;
385
386 bool ulpfec_packet = encapsulated_payload_type == kUlpfecPayloadType;
387 switch (state_) {
388 case kFirstPacket:
389 state_ = kDropEveryOtherPacketUntilUlpfec;
390 break;
391 case kDropEveryOtherPacketUntilUlpfec:
392 if (ulpfec_packet) {
393 state_ = kDropAllMediaPacketsUntilUlpfec;
394 } else if (rtp_packet.SequenceNumber() % 2 == 0) {
395 return DROP_PACKET;
396 }
397 break;
398 case kDropAllMediaPacketsUntilUlpfec:
399 if (!ulpfec_packet)
400 return DROP_PACKET;
401 ulpfec_sequence_number_ = rtp_packet.SequenceNumber();
402 state_ = kDropOneMediaPacket;
403 break;
404 case kDropOneMediaPacket:
405 if (ulpfec_packet)
406 return DROP_PACKET;
407 state_ = kPassOneMediaPacket;
408 return DROP_PACKET;
409 break;
410 case kPassOneMediaPacket:
411 if (ulpfec_packet)
412 return DROP_PACKET;
413 // Pass one media packet after dropped packet after last FEC,
414 // otherwise receiver might never see a seq_no after
415 // |ulpfec_sequence_number_|
416 state_ = kVerifyUlpfecPacketNotInNackList;
417 break;
418 case kVerifyUlpfecPacketNotInNackList:
419 // Continue to drop packets. Make sure no frame can be decoded.
420 if (ulpfec_packet || rtp_packet.SequenceNumber() % 2 == 0)
421 return DROP_PACKET;
422 break;
423 }
424 return SEND_PACKET;
425 }
426
427 Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
428 MutexLock lock_(&mutex_);
429 if (state_ == kVerifyUlpfecPacketNotInNackList) {
430 test::RtcpPacketParser rtcp_parser;
431 rtcp_parser.Parse(packet, length);
432 const std::vector<uint16_t>& nacks = rtcp_parser.nack()->packet_ids();
433 EXPECT_THAT(nacks, Not(Contains(ulpfec_sequence_number_)))
434 << "Got nack for ULPFEC packet";
435 if (!nacks.empty() &&
436 IsNewerSequenceNumber(nacks.back(), ulpfec_sequence_number_)) {
437 observation_complete_.Set();
438 }
439 }
440 return SEND_PACKET;
441 }
442
443 std::unique_ptr<test::PacketTransport> CreateSendTransport(
444 TaskQueueBase* task_queue,
445 Call* sender_call) override {
446 // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
447 // Configure some network delay.
448 const int kNetworkDelayMs = 50;
449 BuiltInNetworkBehaviorConfig config;
450 config.queue_delay_ms = kNetworkDelayMs;
451 return std::make_unique<test::PacketTransport>(
452 task_queue, sender_call, this, test::PacketTransport::kSender,
453 payload_type_map_,
454 std::make_unique<FakeNetworkPipe>(
455 Clock::GetRealTimeClock(),
456 std::make_unique<SimulatedNetwork>(config)));
457 }
458
459 // TODO(holmer): Investigate why we don't send FEC packets when the bitrate
460 // is 10 kbps.
461 void ModifySenderBitrateConfig(
462 BitrateConstraints* bitrate_config) override {
463 const int kMinBitrateBps = 30000;
464 bitrate_config->min_bitrate_bps = kMinBitrateBps;
465 }
466
467 void ModifyVideoConfigs(
468 VideoSendStream::Config* send_config,
469 std::vector<VideoReceiveStream::Config>* receive_configs,
470 VideoEncoderConfig* encoder_config) override {
471 // Configure hybrid NACK/FEC.
472 send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
473 send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
474 send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
475 // Set codec to VP8, otherwise NACK/FEC hybrid will be disabled.
476 send_config->encoder_settings.encoder_factory = &encoder_factory_;
477 send_config->rtp.payload_name = "VP8";
478 send_config->rtp.payload_type = kFakeVideoSendPayloadType;
479 encoder_config->codec_type = kVideoCodecVP8;
480
481 (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
482 (*receive_configs)[0].rtp.red_payload_type = kRedPayloadType;
483 (*receive_configs)[0].rtp.ulpfec_payload_type = kUlpfecPayloadType;
484
485 (*receive_configs)[0].decoders.resize(1);
486 (*receive_configs)[0].decoders[0].payload_type =
487 send_config->rtp.payload_type;
488 (*receive_configs)[0].decoders[0].video_format =
489 SdpVideoFormat(send_config->rtp.payload_name);
490 (*receive_configs)[0].decoders[0].decoder_factory = &decoder_factory_;
491 }
492
493 void PerformTest() override {
494 EXPECT_TRUE(Wait())
495 << "Timed out while waiting for FEC packets to be received.";
496 }
497
498 enum {
499 kFirstPacket,
500 kDropEveryOtherPacketUntilUlpfec,
501 kDropAllMediaPacketsUntilUlpfec,
502 kDropOneMediaPacket,
503 kPassOneMediaPacket,
504 kVerifyUlpfecPacketNotInNackList,
505 } state_;
506
507 Mutex mutex_;
508 uint16_t ulpfec_sequence_number_ RTC_GUARDED_BY(&mutex_);
509 bool has_last_sequence_number_;
510 uint16_t last_sequence_number_;
511 test::FunctionVideoEncoderFactory encoder_factory_;
512 InternalDecoderFactory decoder_factory_;
513 } test;
514
515 RunBaseTest(&test);
516 }
517 } // namespace webrtc
518