1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
12 
13 #include "testing/gtest/include/gtest/gtest.h"
14 
15 namespace webrtc {
16 namespace {
17 static const int kNumSeconds = 10;
18 static const int kWidth = 1920;
19 static const int kHalfWidth = kWidth / 2;
20 static const int kHeight = 1080;
21 static const int kFramerate = 30;
22 static const int kLowQp = 15;
23 static const int kNormalQp = 30;
24 static const int kHighQp = 40;
25 static const int kMaxQp = 56;
26 }  // namespace
27 
28 class QualityScalerTest : public ::testing::Test {
29  public:
30   // Temporal and spatial resolution.
31   struct Resolution {
32     int framerate;
33     int width;
34     int height;
35   };
36 
37  protected:
38   enum ScaleDirection {
39     kKeepScaleAtHighQp,
40     kScaleDown,
41     kScaleDownAboveHighQp,
42     kScaleUp
43   };
44   enum BadQualityMetric { kDropFrame, kReportLowQP };
45 
QualityScalerTest()46   QualityScalerTest() {
47     input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
48                                   kHalfWidth);
49     qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false);
50     qs_.ReportFramerate(kFramerate);
51     qs_.OnEncodeFrame(input_frame_);
52   }
53 
TriggerScale(ScaleDirection scale_direction)54   bool TriggerScale(ScaleDirection scale_direction) {
55     qs_.OnEncodeFrame(input_frame_);
56     int initial_width = qs_.GetScaledResolution().width;
57     for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
58       switch (scale_direction) {
59         case kScaleUp:
60           qs_.ReportQP(kLowQp);
61           break;
62         case kScaleDown:
63           qs_.ReportDroppedFrame();
64           break;
65         case kKeepScaleAtHighQp:
66           qs_.ReportQP(kHighQp);
67           break;
68         case kScaleDownAboveHighQp:
69           qs_.ReportQP(kHighQp + 1);
70           break;
71       }
72       qs_.OnEncodeFrame(input_frame_);
73       if (qs_.GetScaledResolution().width != initial_width)
74         return true;
75     }
76 
77     return false;
78   }
79 
ExpectOriginalFrame()80   void ExpectOriginalFrame() {
81     EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_))
82         << "Using scaled frame instead of original input.";
83   }
84 
ExpectScaleUsingReportedResolution()85   void ExpectScaleUsingReportedResolution() {
86     qs_.OnEncodeFrame(input_frame_);
87     QualityScaler::Resolution res = qs_.GetScaledResolution();
88     const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_);
89     EXPECT_EQ(res.width, scaled_frame.width());
90     EXPECT_EQ(res.height, scaled_frame.height());
91   }
92 
93   void ContinuouslyDownscalesByHalfDimensionsAndBackUp();
94 
95   void DoesNotDownscaleFrameDimensions(int width, int height);
96 
97   Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
98                                      int num_second,
99                                      int initial_framerate);
100 
101   void VerifyQualityAdaptation(int initial_framerate,
102                                int seconds,
103                                bool expect_spatial_resize,
104                                bool expect_framerate_reduction);
105 
106   void DownscaleEndsAt(int input_width,
107                        int input_height,
108                        int end_width,
109                        int end_height);
110 
111   QualityScaler qs_;
112   VideoFrame input_frame_;
113 };
114 
TEST_F(QualityScalerTest,UsesOriginalFrameInitially)115 TEST_F(QualityScalerTest, UsesOriginalFrameInitially) {
116   ExpectOriginalFrame();
117 }
118 
TEST_F(QualityScalerTest,ReportsOriginalResolutionInitially)119 TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) {
120   qs_.OnEncodeFrame(input_frame_);
121   QualityScaler::Resolution res = qs_.GetScaledResolution();
122   EXPECT_EQ(input_frame_.width(), res.width);
123   EXPECT_EQ(input_frame_.height(), res.height);
124 }
125 
TEST_F(QualityScalerTest,DownscalesAfterContinuousFramedrop)126 TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
127   EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds
128                                         << " seconds.";
129   QualityScaler::Resolution res = qs_.GetScaledResolution();
130   EXPECT_LT(res.width, input_frame_.width());
131   EXPECT_LT(res.height, input_frame_.height());
132 }
133 
TEST_F(QualityScalerTest,KeepsScaleAtHighQp)134 TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
135   EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp))
136       << "Downscale at high threshold which should keep scale.";
137   QualityScaler::Resolution res = qs_.GetScaledResolution();
138   EXPECT_EQ(res.width, input_frame_.width());
139   EXPECT_EQ(res.height, input_frame_.height());
140 }
141 
TEST_F(QualityScalerTest,DownscalesAboveHighQp)142 TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
143   EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp))
144       << "No downscale within " << kNumSeconds << " seconds.";
145   QualityScaler::Resolution res = qs_.GetScaledResolution();
146   EXPECT_LT(res.width, input_frame_.width());
147   EXPECT_LT(res.height, input_frame_.height());
148 }
149 
TEST_F(QualityScalerTest,DownscalesAfterTwoThirdsFramedrop)150 TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
151   for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
152     qs_.ReportQP(kNormalQp);
153     qs_.ReportDroppedFrame();
154     qs_.ReportDroppedFrame();
155     qs_.OnEncodeFrame(input_frame_);
156     if (qs_.GetScaledResolution().width < input_frame_.width())
157       return;
158   }
159 
160   FAIL() << "No downscale within " << kNumSeconds << " seconds.";
161 }
162 
TEST_F(QualityScalerTest,DoesNotDownscaleOnNormalQp)163 TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
164   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
165     qs_.ReportQP(kNormalQp);
166     qs_.OnEncodeFrame(input_frame_);
167     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
168         << "Unexpected scale on half framedrop.";
169   }
170 }
171 
TEST_F(QualityScalerTest,DoesNotDownscaleAfterHalfFramedrop)172 TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
173   for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
174     qs_.ReportQP(kNormalQp);
175     qs_.OnEncodeFrame(input_frame_);
176     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
177         << "Unexpected scale on half framedrop.";
178 
179     qs_.ReportDroppedFrame();
180     qs_.OnEncodeFrame(input_frame_);
181     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
182         << "Unexpected scale on half framedrop.";
183   }
184 }
185 
ContinuouslyDownscalesByHalfDimensionsAndBackUp()186 void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
187   const int initial_min_dimension = input_frame_.width() < input_frame_.height()
188                                         ? input_frame_.width()
189                                         : input_frame_.height();
190   int min_dimension = initial_min_dimension;
191   int current_shift = 0;
192   // Drop all frames to force-trigger downscaling.
193   while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) {
194     EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within "
195                                           << kNumSeconds << " seconds.";
196     qs_.OnEncodeFrame(input_frame_);
197     QualityScaler::Resolution res = qs_.GetScaledResolution();
198     min_dimension = res.width < res.height ? res.width : res.height;
199     ++current_shift;
200     ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
201     ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
202     ExpectScaleUsingReportedResolution();
203   }
204 
205   // Make sure we can scale back with good-quality frames.
206   while (min_dimension < initial_min_dimension) {
207     EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds
208                                         << " seconds.";
209     qs_.OnEncodeFrame(input_frame_);
210     QualityScaler::Resolution res = qs_.GetScaledResolution();
211     min_dimension = res.width < res.height ? res.width : res.height;
212     --current_shift;
213     ASSERT_EQ(input_frame_.width() >> current_shift, res.width);
214     ASSERT_EQ(input_frame_.height() >> current_shift, res.height);
215     ExpectScaleUsingReportedResolution();
216   }
217 
218   // Verify we don't start upscaling after further low use.
219   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
220     qs_.ReportQP(kLowQp);
221     ExpectOriginalFrame();
222   }
223 }
224 
TEST_F(QualityScalerTest,ContinuouslyDownscalesByHalfDimensionsAndBackUp)225 TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) {
226   ContinuouslyDownscalesByHalfDimensionsAndBackUp();
227 }
228 
TEST_F(QualityScalerTest,ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp)229 TEST_F(QualityScalerTest,
230        ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) {
231   const int kOddWidth = 517;
232   const int kHalfOddWidth = (kOddWidth + 1) / 2;
233   const int kOddHeight = 1239;
234   input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth,
235                                 kHalfOddWidth);
236   ContinuouslyDownscalesByHalfDimensionsAndBackUp();
237 }
238 
DoesNotDownscaleFrameDimensions(int width,int height)239 void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) {
240   input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2,
241                                 (width + 1) / 2);
242 
243   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
244     qs_.ReportDroppedFrame();
245     qs_.OnEncodeFrame(input_frame_);
246     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width)
247         << "Unexpected scale of minimal-size frame.";
248   }
249 }
250 
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1PxWidth)251 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) {
252   DoesNotDownscaleFrameDimensions(1, kHeight);
253 }
254 
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1PxHeight)255 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) {
256   DoesNotDownscaleFrameDimensions(kWidth, 1);
257 }
258 
TEST_F(QualityScalerTest,DoesNotDownscaleFrom1Px)259 TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) {
260   DoesNotDownscaleFrameDimensions(1, 1);
261 }
262 
TriggerResolutionChange(BadQualityMetric dropframe_lowqp,int num_second,int initial_framerate)263 QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
264     BadQualityMetric dropframe_lowqp,
265     int num_second,
266     int initial_framerate) {
267   QualityScalerTest::Resolution res;
268   res.framerate = initial_framerate;
269   qs_.OnEncodeFrame(input_frame_);
270   res.width = qs_.GetScaledResolution().width;
271   res.height = qs_.GetScaledResolution().height;
272   for (int i = 0; i < kFramerate * num_second; ++i) {
273     switch (dropframe_lowqp) {
274       case kReportLowQP:
275         qs_.ReportQP(kLowQp);
276         break;
277       case kDropFrame:
278         qs_.ReportDroppedFrame();
279         break;
280     }
281     qs_.OnEncodeFrame(input_frame_);
282     // Simulate the case when SetRates is called right after reducing
283     // framerate.
284     qs_.ReportFramerate(initial_framerate);
285     res.framerate = qs_.GetTargetFramerate();
286     if (res.framerate != -1)
287       qs_.ReportFramerate(res.framerate);
288     res.width = qs_.GetScaledResolution().width;
289     res.height = qs_.GetScaledResolution().height;
290   }
291   return res;
292 }
293 
VerifyQualityAdaptation(int initial_framerate,int seconds,bool expect_spatial_resize,bool expect_framerate_reduction)294 void QualityScalerTest::VerifyQualityAdaptation(
295     int initial_framerate,
296     int seconds,
297     bool expect_spatial_resize,
298     bool expect_framerate_reduction) {
299   const int kDisabledBadQpThreshold = kMaxQp + 1;
300   qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
301            kDisabledBadQpThreshold, true);
302   qs_.OnEncodeFrame(input_frame_);
303   int init_width = qs_.GetScaledResolution().width;
304   int init_height = qs_.GetScaledResolution().height;
305 
306   // Test reducing framerate by dropping frame continuously.
307   QualityScalerTest::Resolution res =
308       TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
309 
310   if (expect_framerate_reduction) {
311     EXPECT_LT(res.framerate, initial_framerate);
312   } else {
313     // No framerate reduction, video decimator should be disabled.
314     EXPECT_EQ(-1, res.framerate);
315   }
316 
317   if (expect_spatial_resize) {
318     EXPECT_LT(res.width, init_width);
319     EXPECT_LT(res.height, init_height);
320   } else {
321     EXPECT_EQ(init_width, res.width);
322     EXPECT_EQ(init_height, res.height);
323   }
324 
325   // The "seconds * 1.5" is to ensure spatial resolution to recover.
326   // For example, in 10 seconds test, framerate reduction happens in the first
327   // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
328   // original one. Then it will take only 75 samples to downscale (twice in 150
329   // samples). So to recover the resolution changes, we need more than 10
330   // seconds (i.e, seconds * 1.5). This is because the framerate increases
331   // before spatial size recovers, so it will take 150 samples to recover
332   // spatial size (300 for twice).
333   res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
334   EXPECT_EQ(-1, res.framerate);
335   EXPECT_EQ(init_width, res.width);
336   EXPECT_EQ(init_height, res.height);
337 }
338 
339 // In 5 seconds test, only framerate adjusting should happen.
TEST_F(QualityScalerTest,ChangeFramerateOnly)340 TEST_F(QualityScalerTest, ChangeFramerateOnly) {
341   VerifyQualityAdaptation(kFramerate, 5, false, true);
342 }
343 
344 // In 10 seconds test, framerate adjusting and scaling are both
345 // triggered, it shows that scaling would happen after framerate
346 // adjusting.
TEST_F(QualityScalerTest,ChangeFramerateAndSpatialSize)347 TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
348   VerifyQualityAdaptation(kFramerate, 10, true, true);
349 }
350 
351 // When starting from a low framerate, only spatial size will be changed.
TEST_F(QualityScalerTest,ChangeSpatialSizeOnly)352 TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
353   qs_.ReportFramerate(kFramerate >> 1);
354   VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
355 }
356 
TEST_F(QualityScalerTest,DoesNotDownscaleBelow2xDefaultMinDimensionsWidth)357 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
358   DoesNotDownscaleFrameDimensions(
359       2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000);
360 }
361 
TEST_F(QualityScalerTest,DoesNotDownscaleBelow2xDefaultMinDimensionsHeight)362 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
363   DoesNotDownscaleFrameDimensions(
364       1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1);
365 }
366 
DownscaleEndsAt(int input_width,int input_height,int end_width,int end_height)367 void QualityScalerTest::DownscaleEndsAt(int input_width,
368                                         int input_height,
369                                         int end_width,
370                                         int end_height) {
371   // Create a frame with 2x expected end width/height to verify that we can
372   // scale down to expected end width/height.
373   input_frame_.CreateEmptyFrame(input_width, input_height, input_width,
374                                 (input_width + 1) / 2, (input_width + 1) / 2);
375 
376   int last_width = input_width;
377   int last_height = input_height;
378   // Drop all frames to force-trigger downscaling.
379   while (true) {
380     TriggerScale(kScaleDown);
381     QualityScaler::Resolution res = qs_.GetScaledResolution();
382     if (last_width == res.width) {
383       EXPECT_EQ(last_height, res.height);
384       EXPECT_EQ(end_width, res.width);
385       EXPECT_EQ(end_height, res.height);
386       break;
387     }
388     last_width = res.width;
389     last_height = res.height;
390   }
391 }
392 
TEST_F(QualityScalerTest,DefaultDownscalesTo160x90)393 TEST_F(QualityScalerTest, DefaultDownscalesTo160x90) {
394   DownscaleEndsAt(320, 180, 160, 90);
395 }
396 
TEST_F(QualityScalerTest,DefaultDownscalesTo90x160)397 TEST_F(QualityScalerTest, DefaultDownscalesTo90x160) {
398   DownscaleEndsAt(180, 320, 90, 160);
399 }
400 
TEST_F(QualityScalerTest,DefaultDownscalesFrom1280x720To160x90)401 TEST_F(QualityScalerTest, DefaultDownscalesFrom1280x720To160x90) {
402   DownscaleEndsAt(1280, 720, 160, 90);
403 }
404 
TEST_F(QualityScalerTest,DefaultDoesntDownscaleBelow160x90)405 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow160x90) {
406   DownscaleEndsAt(320 - 1, 180 - 1, 320 - 1, 180 - 1);
407 }
408 
TEST_F(QualityScalerTest,DefaultDoesntDownscaleBelow90x160)409 TEST_F(QualityScalerTest, DefaultDoesntDownscaleBelow90x160) {
410   DownscaleEndsAt(180 - 1, 320 - 1, 180 - 1, 320 - 1);
411 }
412 
TEST_F(QualityScalerTest,RespectsMinResolutionWidth)413 TEST_F(QualityScalerTest, RespectsMinResolutionWidth) {
414   // Should end at 200x100, as width can't go lower.
415   qs_.SetMinResolution(200, 10);
416   DownscaleEndsAt(1600, 800, 200, 100);
417 }
418 
TEST_F(QualityScalerTest,RespectsMinResolutionHeight)419 TEST_F(QualityScalerTest, RespectsMinResolutionHeight) {
420   // Should end at 100x200, as height can't go lower.
421   qs_.SetMinResolution(10, 200);
422   DownscaleEndsAt(800, 1600, 100, 200);
423 }
424 
425 }  // namespace webrtc
426