1 /*
2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <vector>
12 
13 #include "testing/gtest/include/gtest/gtest.h"
14 #include "webrtc/base/scoped_ptr.h"
15 #include "webrtc/common.h"
16 #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h"
17 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
19 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
20 #include "webrtc/modules/video_coding/include/mock/mock_vcm_callbacks.h"
21 #include "webrtc/modules/video_coding/include/video_coding.h"
22 #include "webrtc/modules/video_coding/video_coding_impl.h"
23 #include "webrtc/modules/video_coding/test/test_util.h"
24 #include "webrtc/system_wrappers/include/clock.h"
25 #include "webrtc/test/frame_generator.h"
26 #include "webrtc/test/testsupport/fileutils.h"
27 
28 using ::testing::_;
29 using ::testing::AllOf;
30 using ::testing::ElementsAre;
31 using ::testing::ElementsAreArray;
32 using ::testing::Field;
33 using ::testing::NiceMock;
34 using ::testing::Pointee;
35 using ::testing::Return;
36 using ::testing::FloatEq;
37 using std::vector;
38 using webrtc::test::FrameGenerator;
39 
40 namespace webrtc {
41 namespace vcm {
42 namespace {
43 enum { kMaxNumberOfTemporalLayers = 3 };
44 
45 struct Vp8StreamInfo {
46   float framerate_fps[kMaxNumberOfTemporalLayers];
47   int bitrate_kbps[kMaxNumberOfTemporalLayers];
48 };
49 
50 MATCHER_P(MatchesVp8StreamInfo, expected, "") {
51   bool res = true;
52   for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
53     if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
54       *result_listener << " framerate_fps[" << tl
55                        << "] = " << arg.framerate_fps[tl] << " (expected "
56                        << expected.framerate_fps[tl] << ") ";
57       res = false;
58     }
59     if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
60       *result_listener << " bitrate_kbps[" << tl
61                        << "] = " << arg.bitrate_kbps[tl] << " (expected "
62                        << expected.bitrate_kbps[tl] << ") ";
63       res = false;
64     }
65   }
66   return res;
67 }
68 
69 class EmptyFrameGenerator : public FrameGenerator {
70  public:
EmptyFrameGenerator(int width,int height)71   EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {}
NextFrame()72   VideoFrame* NextFrame() override {
73     frame_.reset(new VideoFrame());
74     frame_->CreateEmptyFrame(width_, height_, width_, (width_ + 1) / 2,
75                              (width_ + 1) / 2);
76     return frame_.get();
77   }
78 
79  private:
80   const int width_;
81   const int height_;
82   rtc::scoped_ptr<VideoFrame> frame_;
83 };
84 
85 class PacketizationCallback : public VCMPacketizationCallback {
86  public:
PacketizationCallback(Clock * clock)87   explicit PacketizationCallback(Clock* clock)
88       : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {}
89 
~PacketizationCallback()90   virtual ~PacketizationCallback() {}
91 
SendData(uint8_t payload_type,const EncodedImage & encoded_image,const RTPFragmentationHeader & fragmentation_header,const RTPVideoHeader * rtp_video_header)92   int32_t SendData(uint8_t payload_type,
93                    const EncodedImage& encoded_image,
94                    const RTPFragmentationHeader& fragmentation_header,
95                    const RTPVideoHeader* rtp_video_header) override {
96     assert(rtp_video_header);
97     frame_data_.push_back(FrameData(encoded_image._length, *rtp_video_header));
98     return 0;
99   }
100 
Reset()101   void Reset() {
102     frame_data_.clear();
103     start_time_ms_ = clock_->TimeInMilliseconds();
104   }
105 
FramerateFpsWithinTemporalLayer(int temporal_layer)106   float FramerateFpsWithinTemporalLayer(int temporal_layer) {
107     return CountFramesWithinTemporalLayer(temporal_layer) *
108            (1000.0 / interval_ms());
109   }
110 
BitrateKbpsWithinTemporalLayer(int temporal_layer)111   float BitrateKbpsWithinTemporalLayer(int temporal_layer) {
112     return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 /
113            interval_ms();
114   }
115 
CalculateVp8StreamInfo()116   Vp8StreamInfo CalculateVp8StreamInfo() {
117     Vp8StreamInfo info;
118     for (int tl = 0; tl < 3; ++tl) {
119       info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
120       info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
121     }
122     return info;
123   }
124 
125  private:
126   struct FrameData {
FrameDatawebrtc::vcm::__anon2f36a7e80111::PacketizationCallback::FrameData127     FrameData() {}
128 
FrameDatawebrtc::vcm::__anon2f36a7e80111::PacketizationCallback::FrameData129     FrameData(size_t payload_size, const RTPVideoHeader& rtp_video_header)
130         : payload_size(payload_size), rtp_video_header(rtp_video_header) {}
131 
132     size_t payload_size;
133     RTPVideoHeader rtp_video_header;
134   };
135 
interval_ms()136   int64_t interval_ms() {
137     int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_);
138     EXPECT_GT(diff, 0);
139     return diff;
140   }
141 
CountFramesWithinTemporalLayer(int temporal_layer)142   int CountFramesWithinTemporalLayer(int temporal_layer) {
143     int frames = 0;
144     for (size_t i = 0; i < frame_data_.size(); ++i) {
145       EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
146       const uint8_t temporal_idx =
147           frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
148       if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
149         frames++;
150     }
151     return frames;
152   }
153 
SumPayloadBytesWithinTemporalLayer(int temporal_layer)154   size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) {
155     size_t payload_size = 0;
156     for (size_t i = 0; i < frame_data_.size(); ++i) {
157       EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
158       const uint8_t temporal_idx =
159           frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
160       if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
161         payload_size += frame_data_[i].payload_size;
162     }
163     return payload_size;
164   }
165 
166   Clock* clock_;
167   int64_t start_time_ms_;
168   vector<FrameData> frame_data_;
169 };
170 
171 class TestVideoSender : public ::testing::Test {
172  protected:
173   // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
174   // a special case (e.g. frame rate in media optimization).
TestVideoSender()175   TestVideoSender() : clock_(1000), packetization_callback_(&clock_) {}
176 
SetUp()177   void SetUp() override {
178     sender_.reset(
179         new VideoSender(&clock_, &post_encode_callback_, nullptr, nullptr));
180     EXPECT_EQ(0, sender_->RegisterTransportCallback(&packetization_callback_));
181   }
182 
AddFrame()183   void AddFrame() {
184     assert(generator_.get());
185     sender_->AddVideoFrame(*generator_->NextFrame(), NULL, NULL);
186   }
187 
188   SimulatedClock clock_;
189   PacketizationCallback packetization_callback_;
190   MockEncodedImageCallback post_encode_callback_;
191   // Used by subclassing tests, need to outlive sender_.
192   rtc::scoped_ptr<VideoEncoder> encoder_;
193   rtc::scoped_ptr<VideoSender> sender_;
194   rtc::scoped_ptr<FrameGenerator> generator_;
195 };
196 
197 class TestVideoSenderWithMockEncoder : public TestVideoSender {
198  protected:
199   static const int kDefaultWidth = 1280;
200   static const int kDefaultHeight = 720;
201   static const int kNumberOfStreams = 3;
202   static const int kNumberOfLayers = 3;
203   static const int kUnusedPayloadType = 10;
204 
SetUp()205   void SetUp() override {
206     TestVideoSender::SetUp();
207     sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, false);
208     VideoCodingModule::Codec(kVideoCodecVP8, &settings_);
209     settings_.numberOfSimulcastStreams = kNumberOfStreams;
210     ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100,
211                     &settings_.simulcastStream[0]);
212     ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500,
213                     &settings_.simulcastStream[1]);
214     ConfigureStream(kDefaultWidth, kDefaultHeight, 1200,
215                     &settings_.simulcastStream[2]);
216     settings_.plType = kUnusedPayloadType;  // Use the mocked encoder.
217     generator_.reset(
218         new EmptyFrameGenerator(settings_.width, settings_.height));
219     EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
220   }
221 
TearDown()222   void TearDown() override { sender_.reset(); }
223 
ExpectIntraRequest(int stream)224   void ExpectIntraRequest(int stream) {
225     if (stream == -1) {
226       // No intra request expected.
227       EXPECT_CALL(
228           encoder_,
229           Encode(_, _, Pointee(ElementsAre(kVideoFrameDelta, kVideoFrameDelta,
230                                            kVideoFrameDelta))))
231           .Times(1)
232           .WillRepeatedly(Return(0));
233       return;
234     }
235     assert(stream >= 0);
236     assert(stream < kNumberOfStreams);
237     std::vector<FrameType> frame_types(kNumberOfStreams, kVideoFrameDelta);
238     frame_types[stream] = kVideoFrameKey;
239     EXPECT_CALL(encoder_,
240                 Encode(_, _, Pointee(ElementsAreArray(&frame_types[0],
241                                                       frame_types.size()))))
242         .Times(1)
243         .WillRepeatedly(Return(0));
244   }
245 
ConfigureStream(int width,int height,int max_bitrate,SimulcastStream * stream)246   static void ConfigureStream(int width,
247                               int height,
248                               int max_bitrate,
249                               SimulcastStream* stream) {
250     assert(stream);
251     stream->width = width;
252     stream->height = height;
253     stream->maxBitrate = max_bitrate;
254     stream->numberOfTemporalLayers = kNumberOfLayers;
255     stream->qpMax = 45;
256   }
257 
258   VideoCodec settings_;
259   NiceMock<MockVideoEncoder> encoder_;
260 };
261 
TEST_F(TestVideoSenderWithMockEncoder,TestIntraRequests)262 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) {
263   EXPECT_EQ(0, sender_->IntraFrameRequest(0));
264   ExpectIntraRequest(0);
265   AddFrame();
266   ExpectIntraRequest(-1);
267   AddFrame();
268 
269   EXPECT_EQ(0, sender_->IntraFrameRequest(1));
270   ExpectIntraRequest(1);
271   AddFrame();
272   ExpectIntraRequest(-1);
273   AddFrame();
274 
275   EXPECT_EQ(0, sender_->IntraFrameRequest(2));
276   ExpectIntraRequest(2);
277   AddFrame();
278   ExpectIntraRequest(-1);
279   AddFrame();
280 
281   EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
282   ExpectIntraRequest(-1);
283   AddFrame();
284 
285   EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
286   ExpectIntraRequest(-1);
287   AddFrame();
288 }
289 
TEST_F(TestVideoSenderWithMockEncoder,TestIntraRequestsInternalCapture)290 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) {
291   // De-register current external encoder.
292   sender_->RegisterExternalEncoder(nullptr, kUnusedPayloadType, false);
293   // Register encoder with internal capture.
294   sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, true);
295   EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
296   ExpectIntraRequest(0);
297   EXPECT_EQ(0, sender_->IntraFrameRequest(0));
298   ExpectIntraRequest(1);
299   EXPECT_EQ(0, sender_->IntraFrameRequest(1));
300   ExpectIntraRequest(2);
301   EXPECT_EQ(0, sender_->IntraFrameRequest(2));
302   // No requests expected since these indices are out of bounds.
303   EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
304   EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
305 }
306 
TEST_F(TestVideoSenderWithMockEncoder,EncoderFramerateUpdatedViaProcess)307 TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) {
308   sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200);
309   const int64_t kRateStatsWindowMs = 2000;
310   const uint32_t kInputFps = 20;
311   int64_t start_time = clock_.TimeInMilliseconds();
312   while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
313     AddFrame();
314     clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
315   }
316   EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
317   sender_->Process();
318   AddFrame();
319 }
320 
TEST_F(TestVideoSenderWithMockEncoder,NoRedundantSetChannelParameterOrSetRatesCalls)321 TEST_F(TestVideoSenderWithMockEncoder,
322        NoRedundantSetChannelParameterOrSetRatesCalls) {
323   const uint8_t kLossRate = 4;
324   const uint8_t kRtt = 200;
325   const int64_t kRateStatsWindowMs = 2000;
326   const uint32_t kInputFps = 20;
327   int64_t start_time = clock_.TimeInMilliseconds();
328   // Expect initial call to SetChannelParameters. Rates are initialized through
329   // InitEncode and expects no additional call before the framerate (or bitrate)
330   // updates.
331   EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt))
332       .Times(1)
333       .WillOnce(Return(0));
334   sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt);
335   while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
336     AddFrame();
337     clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
338   }
339   // After process, input framerate should be updated but not ChannelParameters
340   // as they are the same as before.
341   EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
342   sender_->Process();
343   AddFrame();
344   // Call to SetChannelParameters with changed bitrate should call encoder
345   // SetRates but not encoder SetChannelParameters (that are unchanged).
346   EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps))
347       .Times(1)
348       .WillOnce(Return(0));
349   sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate,
350                                 kRtt);
351   AddFrame();
352 }
353 
354 class TestVideoSenderWithVp8 : public TestVideoSender {
355  public:
TestVideoSenderWithVp8()356   TestVideoSenderWithVp8()
357       : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {}
358 
SetUp()359   void SetUp() override {
360     TestVideoSender::SetUp();
361 
362     const char* input_video = "foreman_cif";
363     const int width = 352;
364     const int height = 288;
365     generator_.reset(FrameGenerator::CreateFromYuvFile(
366         std::vector<std::string>(1, test::ResourcePath(input_video, "yuv")),
367         width, height, 1));
368 
369     codec_ = MakeVp8VideoCodec(width, height, 3);
370     codec_.minBitrate = 10;
371     codec_.startBitrate = codec_bitrate_kbps_;
372     codec_.maxBitrate = codec_bitrate_kbps_;
373     encoder_.reset(VP8Encoder::Create());
374     sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false);
375     EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200));
376   }
377 
MakeVp8VideoCodec(int width,int height,int temporal_layers)378   static VideoCodec MakeVp8VideoCodec(int width,
379                                       int height,
380                                       int temporal_layers) {
381     VideoCodec codec;
382     VideoCodingModule::Codec(kVideoCodecVP8, &codec);
383     codec.width = width;
384     codec.height = height;
385     codec.codecSpecific.VP8.numberOfTemporalLayers = temporal_layers;
386     return codec;
387   }
388 
InsertFrames(float framerate,float seconds)389   void InsertFrames(float framerate, float seconds) {
390     for (int i = 0; i < seconds * framerate; ++i) {
391       clock_.AdvanceTimeMilliseconds(1000.0f / framerate);
392       EXPECT_CALL(post_encode_callback_, Encoded(_, NULL, NULL))
393           .WillOnce(Return(0));
394       AddFrame();
395       // SetChannelParameters needs to be called frequently to propagate
396       // framerate from the media optimization into the encoder.
397       // Note: SetChannelParameters fails if less than 2 frames are in the
398       // buffer since it will fail to calculate the framerate.
399       if (i != 0) {
400         EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
401                               available_bitrate_kbps_ * 1000, 0, 200));
402       }
403     }
404   }
405 
SimulateWithFramerate(float framerate)406   Vp8StreamInfo SimulateWithFramerate(float framerate) {
407     const float short_simulation_interval = 5.0;
408     const float long_simulation_interval = 10.0;
409     // It appears that this 5 seconds simulation is needed to allow
410     // bitrate and framerate to stabilize.
411     InsertFrames(framerate, short_simulation_interval);
412     packetization_callback_.Reset();
413 
414     InsertFrames(framerate, long_simulation_interval);
415     return packetization_callback_.CalculateVp8StreamInfo();
416   }
417 
418  protected:
419   VideoCodec codec_;
420   int codec_bitrate_kbps_;
421   int available_bitrate_kbps_;
422 };
423 
424 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
425 #define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy
426 #else
427 #define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
428 #endif
TEST_F(TestVideoSenderWithVp8,MAYBE_FixedTemporalLayersStrategy)429 TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
430   const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
431   const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
432   const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
433   {
434     Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
435     EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
436   }
437   {
438     Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
439     EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
440   }
441 }
442 
443 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
444 #define MAYBE_RealTimeTemporalLayersStrategy \
445   DISABLED_RealTimeTemporalLayersStrategy
446 #else
447 #define MAYBE_RealTimeTemporalLayersStrategy RealTimeTemporalLayersStrategy
448 #endif
TEST_F(TestVideoSenderWithVp8,MAYBE_RealTimeTemporalLayersStrategy)449 TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) {
450   Config extra_options;
451   extra_options.Set<TemporalLayers::Factory>(
452       new RealTimeTemporalLayersFactory());
453   VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
454   codec.extra_options = &extra_options;
455   codec.minBitrate = 10;
456   codec.startBitrate = codec_bitrate_kbps_;
457   codec.maxBitrate = codec_bitrate_kbps_;
458   EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
459 
460   const int low_b = codec_bitrate_kbps_ * 0.4;
461   const int mid_b = codec_bitrate_kbps_ * 0.6;
462   const int high_b = codec_bitrate_kbps_;
463 
464   {
465     Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
466     EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
467   }
468   {
469     Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
470     EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
471   }
472   {
473     Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
474     EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
475   }
476   {
477     Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
478     EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
479   }
480   {
481     // TODO(andresp): Find out why this fails with framerate = 7.5
482     Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
483     EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
484   }
485 }
486 }  // namespace
487 }  // namespace vcm
488 }  // namespace webrtc
489