1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/scenario/stats_collection.h"
11
12 #include "test/gtest.h"
13 #include "test/scenario/scenario.h"
14
15 namespace webrtc {
16 namespace test {
17 namespace {
CreateAnalyzedStream(Scenario * s,NetworkSimulationConfig network_config,VideoQualityAnalyzer * analyzer,CallStatsCollectors * collectors)18 void CreateAnalyzedStream(Scenario* s,
19 NetworkSimulationConfig network_config,
20 VideoQualityAnalyzer* analyzer,
21 CallStatsCollectors* collectors) {
22 VideoStreamConfig config;
23 config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
24 config.encoder.implementation =
25 VideoStreamConfig::Encoder::Implementation::kSoftware;
26 config.hooks.frame_pair_handlers = {analyzer->Handler()};
27 auto* caller = s->CreateClient("caller", CallClientConfig());
28 auto* callee = s->CreateClient("callee", CallClientConfig());
29 auto route =
30 s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)}, callee,
31 {s->CreateSimulationNode(NetworkSimulationConfig())});
32 VideoStreamPair* video = s->CreateVideoStream(route->forward(), config);
33 auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig());
34 s->Every(TimeDelta::Seconds(1), [=] {
35 collectors->call.AddStats(caller->GetStats());
36 collectors->video_send.AddStats(video->send()->GetStats(), s->Now());
37 collectors->audio_receive.AddStats(audio->receive()->GetStats());
38
39 // Querying the video stats from within the expected runtime environment
40 // (i.e. the TQ that belongs to the CallClient, not the Scenario TQ that
41 // we're currently on).
42 VideoReceiveStream::Stats video_receive_stats;
43 auto* video_stream = video->receive();
44 callee->SendTask([&video_stream, &video_receive_stats]() {
45 video_receive_stats = video_stream->GetStats();
46 });
47 collectors->video_receive.AddStats(video_receive_stats);
48 });
49 }
50 } // namespace
51
TEST(ScenarioAnalyzerTest,PsnrIsHighWhenNetworkIsGood)52 TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) {
53 VideoQualityAnalyzer analyzer;
54 CallStatsCollectors stats;
55 {
56 Scenario s;
57 NetworkSimulationConfig good_network;
58 good_network.bandwidth = DataRate::KilobitsPerSec(1000);
59 CreateAnalyzedStream(&s, good_network, &analyzer, &stats);
60 s.RunFor(TimeDelta::Seconds(3));
61 }
62 // This is a change detecting test, the targets are based on previous runs and
63 // might change due to changes in configuration and encoder etc. The main
64 // purpose is to show how the stats can be used. To avoid being overly
65 // sensistive to change, the ranges are chosen to be quite large.
66 EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 43, 10);
67 EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300);
68 EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200);
69 EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
70 EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20);
71 }
72
TEST(ScenarioAnalyzerTest,PsnrIsLowWhenNetworkIsBad)73 TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) {
74 VideoQualityAnalyzer analyzer;
75 CallStatsCollectors stats;
76 {
77 Scenario s;
78 NetworkSimulationConfig bad_network;
79 bad_network.bandwidth = DataRate::KilobitsPerSec(100);
80 bad_network.loss_rate = 0.02;
81 CreateAnalyzedStream(&s, bad_network, &analyzer, &stats);
82 s.RunFor(TimeDelta::Seconds(3));
83 }
84 // This is a change detecting test, the targets are based on previous runs and
85 // might change due to changes in configuration and encoder etc.
86 EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 20, 10);
87 EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50);
88 EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50);
89 EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
90 EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 200, 150);
91 }
92
TEST(ScenarioAnalyzerTest,CountsCapturedButNotRendered)93 TEST(ScenarioAnalyzerTest, CountsCapturedButNotRendered) {
94 VideoQualityAnalyzer analyzer;
95 CallStatsCollectors stats;
96 {
97 Scenario s;
98 NetworkSimulationConfig long_delays;
99 long_delays.delay = TimeDelta::Seconds(5);
100 CreateAnalyzedStream(&s, long_delays, &analyzer, &stats);
101 // Enough time to send frames but not enough to deliver.
102 s.RunFor(TimeDelta::Millis(100));
103 }
104 EXPECT_GE(analyzer.stats().capture.count, 1);
105 EXPECT_EQ(analyzer.stats().render.count, 0);
106 }
107 } // namespace test
108 } // namespace webrtc
109