1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "video/video_analyzer.h"
11
12 #include <algorithm>
13 #include <utility>
14
15 #include "absl/algorithm/container.h"
16 #include "absl/flags/flag.h"
17 #include "absl/flags/parse.h"
18 #include "common_video/libyuv/include/webrtc_libyuv.h"
19 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
20 #include "modules/rtp_rtcp/source/rtp_packet.h"
21 #include "rtc_base/cpu_time.h"
22 #include "rtc_base/format_macros.h"
23 #include "rtc_base/memory_usage.h"
24 #include "rtc_base/task_queue_for_test.h"
25 #include "rtc_base/task_utils/repeating_task.h"
26 #include "system_wrappers/include/cpu_info.h"
27 #include "test/call_test.h"
28 #include "test/testsupport/file_utils.h"
29 #include "test/testsupport/frame_writer.h"
30 #include "test/testsupport/perf_test.h"
31 #include "test/testsupport/test_artifacts.h"
32
33 ABSL_FLAG(bool,
34 save_worst_frame,
35 false,
36 "Enable saving a frame with the lowest PSNR to a jpeg file in the "
37 "test_artifacts_dir");
38
39 namespace webrtc {
40 namespace {
41 constexpr TimeDelta kSendStatsPollingInterval = TimeDelta::Seconds(1);
42 constexpr size_t kMaxComparisons = 10;
43 // How often is keep alive message printed.
44 constexpr int kKeepAliveIntervalSeconds = 30;
45 // Interval between checking that the test is over.
46 constexpr int kProbingIntervalMs = 500;
47 constexpr int kKeepAliveIntervalIterations =
48 kKeepAliveIntervalSeconds * 1000 / kProbingIntervalMs;
49
IsFlexfec(int payload_type)50 bool IsFlexfec(int payload_type) {
51 return payload_type == test::CallTest::kFlexfecPayloadType;
52 }
53 } // namespace
54
VideoAnalyzer(test::LayerFilteringTransport * transport,const std::string & test_label,double avg_psnr_threshold,double avg_ssim_threshold,int duration_frames,TimeDelta test_duration,FILE * graph_data_output_file,const std::string & graph_title,uint32_t ssrc_to_analyze,uint32_t rtx_ssrc_to_analyze,size_t selected_stream,int selected_sl,int selected_tl,bool is_quick_test_enabled,Clock * clock,std::string rtp_dump_name,TaskQueueBase * task_queue)55 VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
56 const std::string& test_label,
57 double avg_psnr_threshold,
58 double avg_ssim_threshold,
59 int duration_frames,
60 TimeDelta test_duration,
61 FILE* graph_data_output_file,
62 const std::string& graph_title,
63 uint32_t ssrc_to_analyze,
64 uint32_t rtx_ssrc_to_analyze,
65 size_t selected_stream,
66 int selected_sl,
67 int selected_tl,
68 bool is_quick_test_enabled,
69 Clock* clock,
70 std::string rtp_dump_name,
71 TaskQueueBase* task_queue)
72 : transport_(transport),
73 receiver_(nullptr),
74 call_(nullptr),
75 send_stream_(nullptr),
76 receive_stream_(nullptr),
77 audio_receive_stream_(nullptr),
78 captured_frame_forwarder_(this, clock, duration_frames, test_duration),
79 test_label_(test_label),
80 graph_data_output_file_(graph_data_output_file),
81 graph_title_(graph_title),
82 ssrc_to_analyze_(ssrc_to_analyze),
83 rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
84 selected_stream_(selected_stream),
85 selected_sl_(selected_sl),
86 selected_tl_(selected_tl),
87 mean_decode_time_ms_(0.0),
88 freeze_count_(0),
89 total_freezes_duration_ms_(0),
90 total_frames_duration_ms_(0),
91 sum_squared_frame_durations_(0),
92 decode_frame_rate_(0),
93 render_frame_rate_(0),
94 last_fec_bytes_(0),
95 frames_to_process_(duration_frames),
96 test_end_(clock->CurrentTime() + test_duration),
97 frames_recorded_(0),
98 frames_processed_(0),
99 captured_frames_(0),
100 dropped_frames_(0),
101 dropped_frames_before_first_encode_(0),
102 dropped_frames_before_rendering_(0),
103 last_render_time_(0),
104 last_render_delta_ms_(0),
105 last_unfreeze_time_ms_(0),
106 rtp_timestamp_delta_(0),
107 cpu_time_(0),
108 wallclock_time_(0),
109 avg_psnr_threshold_(avg_psnr_threshold),
110 avg_ssim_threshold_(avg_ssim_threshold),
111 is_quick_test_enabled_(is_quick_test_enabled),
112 quit_(false),
113 done_(true, false),
114 vp8_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP8)),
115 vp9_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP9)),
116 clock_(clock),
117 start_ms_(clock->TimeInMilliseconds()),
118 task_queue_(task_queue) {
119 // Create thread pool for CPU-expensive PSNR/SSIM calculations.
120
121 // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
122 // so that we don't accidentally starve "real" worker threads (codec etc).
123 // Also, don't allocate more than kMaxComparisonThreads, even if there are
124 // spare cores.
125
126 uint32_t num_cores = CpuInfo::DetectNumberOfCores();
127 RTC_DCHECK_GE(num_cores, 1);
128 static const uint32_t kMinCoresLeft = 4;
129 static const uint32_t kMaxComparisonThreads = 8;
130
131 if (num_cores <= kMinCoresLeft) {
132 num_cores = 1;
133 } else {
134 num_cores -= kMinCoresLeft;
135 num_cores = std::min(num_cores, kMaxComparisonThreads);
136 }
137
138 for (uint32_t i = 0; i < num_cores; ++i) {
139 rtc::PlatformThread* thread =
140 new rtc::PlatformThread(&FrameComparisonThread, this, "Analyzer");
141 thread->Start();
142 comparison_thread_pool_.push_back(thread);
143 }
144
145 if (!rtp_dump_name.empty()) {
146 fprintf(stdout, "Writing rtp dump to %s\n", rtp_dump_name.c_str());
147 rtp_file_writer_.reset(test::RtpFileWriter::Create(
148 test::RtpFileWriter::kRtpDump, rtp_dump_name));
149 }
150 }
151
~VideoAnalyzer()152 VideoAnalyzer::~VideoAnalyzer() {
153 {
154 MutexLock lock(&comparison_lock_);
155 quit_ = true;
156 }
157 for (rtc::PlatformThread* thread : comparison_thread_pool_) {
158 thread->Stop();
159 delete thread;
160 }
161 }
162
SetReceiver(PacketReceiver * receiver)163 void VideoAnalyzer::SetReceiver(PacketReceiver* receiver) {
164 receiver_ = receiver;
165 }
166
SetSource(rtc::VideoSourceInterface<VideoFrame> * video_source,bool respect_sink_wants)167 void VideoAnalyzer::SetSource(
168 rtc::VideoSourceInterface<VideoFrame>* video_source,
169 bool respect_sink_wants) {
170 if (respect_sink_wants)
171 captured_frame_forwarder_.SetSource(video_source);
172 rtc::VideoSinkWants wants;
173 video_source->AddOrUpdateSink(InputInterface(), wants);
174 }
175
SetCall(Call * call)176 void VideoAnalyzer::SetCall(Call* call) {
177 MutexLock lock(&lock_);
178 RTC_DCHECK(!call_);
179 call_ = call;
180 }
181
SetSendStream(VideoSendStream * stream)182 void VideoAnalyzer::SetSendStream(VideoSendStream* stream) {
183 MutexLock lock(&lock_);
184 RTC_DCHECK(!send_stream_);
185 send_stream_ = stream;
186 }
187
SetReceiveStream(VideoReceiveStream * stream)188 void VideoAnalyzer::SetReceiveStream(VideoReceiveStream* stream) {
189 MutexLock lock(&lock_);
190 RTC_DCHECK(!receive_stream_);
191 receive_stream_ = stream;
192 }
193
SetAudioReceiveStream(AudioReceiveStream * recv_stream)194 void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) {
195 MutexLock lock(&lock_);
196 RTC_CHECK(!audio_receive_stream_);
197 audio_receive_stream_ = recv_stream;
198 }
199
InputInterface()200 rtc::VideoSinkInterface<VideoFrame>* VideoAnalyzer::InputInterface() {
201 return &captured_frame_forwarder_;
202 }
203
OutputInterface()204 rtc::VideoSourceInterface<VideoFrame>* VideoAnalyzer::OutputInterface() {
205 return &captured_frame_forwarder_;
206 }
207
DeliverPacket(MediaType media_type,rtc::CopyOnWriteBuffer packet,int64_t packet_time_us)208 PacketReceiver::DeliveryStatus VideoAnalyzer::DeliverPacket(
209 MediaType media_type,
210 rtc::CopyOnWriteBuffer packet,
211 int64_t packet_time_us) {
212 // Ignore timestamps of RTCP packets. They're not synchronized with
213 // RTP packet timestamps and so they would confuse wrap_handler_.
214 if (RtpHeaderParser::IsRtcp(packet.cdata(), packet.size())) {
215 return receiver_->DeliverPacket(media_type, std::move(packet),
216 packet_time_us);
217 }
218
219 if (rtp_file_writer_) {
220 test::RtpPacket p;
221 memcpy(p.data, packet.cdata(), packet.size());
222 p.length = packet.size();
223 p.original_length = packet.size();
224 p.time_ms = clock_->TimeInMilliseconds() - start_ms_;
225 rtp_file_writer_->WritePacket(&p);
226 }
227
228 RtpPacket rtp_packet;
229 rtp_packet.Parse(packet);
230 if (!IsFlexfec(rtp_packet.PayloadType()) &&
231 (rtp_packet.Ssrc() == ssrc_to_analyze_ ||
232 rtp_packet.Ssrc() == rtx_ssrc_to_analyze_)) {
233 // Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
234 // (FlexFEC and media are sent on different SSRCs, which have different
235 // timestamps spaces.)
236 // Also ignore packets from wrong SSRC, but include retransmits.
237 MutexLock lock(&lock_);
238 int64_t timestamp =
239 wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
240 recv_times_[timestamp] = clock_->CurrentNtpInMilliseconds();
241 }
242
243 return receiver_->DeliverPacket(media_type, std::move(packet),
244 packet_time_us);
245 }
246
PreEncodeOnFrame(const VideoFrame & video_frame)247 void VideoAnalyzer::PreEncodeOnFrame(const VideoFrame& video_frame) {
248 MutexLock lock(&lock_);
249 if (!first_encoded_timestamp_) {
250 while (frames_.front().timestamp() != video_frame.timestamp()) {
251 ++dropped_frames_before_first_encode_;
252 frames_.pop_front();
253 RTC_CHECK(!frames_.empty());
254 }
255 first_encoded_timestamp_ = video_frame.timestamp();
256 }
257 }
258
PostEncodeOnFrame(size_t stream_id,uint32_t timestamp)259 void VideoAnalyzer::PostEncodeOnFrame(size_t stream_id, uint32_t timestamp) {
260 MutexLock lock(&lock_);
261 if (!first_sent_timestamp_ && stream_id == selected_stream_) {
262 first_sent_timestamp_ = timestamp;
263 }
264 }
265
SendRtp(const uint8_t * packet,size_t length,const PacketOptions & options)266 bool VideoAnalyzer::SendRtp(const uint8_t* packet,
267 size_t length,
268 const PacketOptions& options) {
269 RtpPacket rtp_packet;
270 rtp_packet.Parse(packet, length);
271
272 int64_t current_time = clock_->CurrentNtpInMilliseconds();
273
274 bool result = transport_->SendRtp(packet, length, options);
275 {
276 MutexLock lock(&lock_);
277 if (rtp_timestamp_delta_ == 0 && rtp_packet.Ssrc() == ssrc_to_analyze_) {
278 RTC_CHECK(static_cast<bool>(first_sent_timestamp_));
279 rtp_timestamp_delta_ = rtp_packet.Timestamp() - *first_sent_timestamp_;
280 }
281
282 if (!IsFlexfec(rtp_packet.PayloadType()) &&
283 rtp_packet.Ssrc() == ssrc_to_analyze_) {
284 // Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
285 // (FlexFEC and media are sent on different SSRCs, which have different
286 // timestamps spaces.)
287 // Also ignore packets from wrong SSRC and retransmits.
288 int64_t timestamp =
289 wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
290 send_times_[timestamp] = current_time;
291
292 if (IsInSelectedSpatialAndTemporalLayer(rtp_packet)) {
293 encoded_frame_sizes_[timestamp] += rtp_packet.payload_size();
294 }
295 }
296 }
297 return result;
298 }
299
SendRtcp(const uint8_t * packet,size_t length)300 bool VideoAnalyzer::SendRtcp(const uint8_t* packet, size_t length) {
301 return transport_->SendRtcp(packet, length);
302 }
303
OnFrame(const VideoFrame & video_frame)304 void VideoAnalyzer::OnFrame(const VideoFrame& video_frame) {
305 int64_t render_time_ms = clock_->CurrentNtpInMilliseconds();
306
307 MutexLock lock(&lock_);
308
309 StartExcludingCpuThreadTime();
310
311 int64_t send_timestamp =
312 wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_);
313
314 while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) {
315 if (!last_rendered_frame_) {
316 // No previous frame rendered, this one was dropped after sending but
317 // before rendering.
318 ++dropped_frames_before_rendering_;
319 } else {
320 AddFrameComparison(frames_.front(), *last_rendered_frame_, true,
321 render_time_ms);
322 }
323 frames_.pop_front();
324 RTC_DCHECK(!frames_.empty());
325 }
326
327 VideoFrame reference_frame = frames_.front();
328 frames_.pop_front();
329 int64_t reference_timestamp =
330 wrap_handler_.Unwrap(reference_frame.timestamp());
331 if (send_timestamp == reference_timestamp - 1) {
332 // TODO(ivica): Make this work for > 2 streams.
333 // Look at RTPSender::BuildRTPHeader.
334 ++send_timestamp;
335 }
336 ASSERT_EQ(reference_timestamp, send_timestamp);
337
338 AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
339
340 last_rendered_frame_ = video_frame;
341
342 StopExcludingCpuThreadTime();
343 }
344
Wait()345 void VideoAnalyzer::Wait() {
346 // Frame comparisons can be very expensive. Wait for test to be done, but
347 // at time-out check if frames_processed is going up. If so, give it more
348 // time, otherwise fail. Hopefully this will reduce test flakiness.
349
350 RepeatingTaskHandle stats_polling_task = RepeatingTaskHandle::DelayedStart(
351 task_queue_, kSendStatsPollingInterval, [this] {
352 PollStats();
353 return kSendStatsPollingInterval;
354 });
355
356 int last_frames_processed = -1;
357 int last_frames_captured = -1;
358 int iteration = 0;
359
360 while (!done_.Wait(kProbingIntervalMs)) {
361 int frames_processed;
362 int frames_captured;
363 {
364 MutexLock lock(&comparison_lock_);
365 frames_processed = frames_processed_;
366 frames_captured = captured_frames_;
367 }
368
369 // Print some output so test infrastructure won't think we've crashed.
370 const char* kKeepAliveMessages[3] = {
371 "Uh, I'm-I'm not quite dead, sir.",
372 "Uh, I-I think uh, I could pull through, sir.",
373 "Actually, I think I'm all right to come with you--"};
374 if (++iteration % kKeepAliveIntervalIterations == 0) {
375 printf("- %s\n", kKeepAliveMessages[iteration % 3]);
376 }
377
378 if (last_frames_processed == -1) {
379 last_frames_processed = frames_processed;
380 last_frames_captured = frames_captured;
381 continue;
382 }
383 if (frames_processed == last_frames_processed &&
384 last_frames_captured == frames_captured &&
385 clock_->CurrentTime() > test_end_) {
386 done_.Set();
387 break;
388 }
389 last_frames_processed = frames_processed;
390 last_frames_captured = frames_captured;
391 }
392
393 if (iteration > 0)
394 printf("- Farewell, sweet Concorde!\n");
395
396 SendTask(RTC_FROM_HERE, task_queue_, [&] { stats_polling_task.Stop(); });
397
398 PrintResults();
399 if (graph_data_output_file_)
400 PrintSamplesToFile();
401 }
402
StartMeasuringCpuProcessTime()403 void VideoAnalyzer::StartMeasuringCpuProcessTime() {
404 MutexLock lock(&cpu_measurement_lock_);
405 cpu_time_ -= rtc::GetProcessCpuTimeNanos();
406 wallclock_time_ -= rtc::SystemTimeNanos();
407 }
408
StopMeasuringCpuProcessTime()409 void VideoAnalyzer::StopMeasuringCpuProcessTime() {
410 MutexLock lock(&cpu_measurement_lock_);
411 cpu_time_ += rtc::GetProcessCpuTimeNanos();
412 wallclock_time_ += rtc::SystemTimeNanos();
413 }
414
StartExcludingCpuThreadTime()415 void VideoAnalyzer::StartExcludingCpuThreadTime() {
416 MutexLock lock(&cpu_measurement_lock_);
417 cpu_time_ += rtc::GetThreadCpuTimeNanos();
418 }
419
StopExcludingCpuThreadTime()420 void VideoAnalyzer::StopExcludingCpuThreadTime() {
421 MutexLock lock(&cpu_measurement_lock_);
422 cpu_time_ -= rtc::GetThreadCpuTimeNanos();
423 }
424
GetCpuUsagePercent()425 double VideoAnalyzer::GetCpuUsagePercent() {
426 MutexLock lock(&cpu_measurement_lock_);
427 return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
428 }
429
IsInSelectedSpatialAndTemporalLayer(const RtpPacket & rtp_packet)430 bool VideoAnalyzer::IsInSelectedSpatialAndTemporalLayer(
431 const RtpPacket& rtp_packet) {
432 if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP8) {
433 auto parsed_payload = vp8_depacketizer_->Parse(rtp_packet.PayloadBuffer());
434 RTC_DCHECK(parsed_payload);
435 const auto& vp8_header = absl::get<RTPVideoHeaderVP8>(
436 parsed_payload->video_header.video_type_header);
437 int temporal_idx = vp8_header.temporalIdx;
438 return selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
439 temporal_idx <= selected_tl_;
440 }
441
442 if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP9) {
443 auto parsed_payload = vp9_depacketizer_->Parse(rtp_packet.PayloadBuffer());
444 RTC_DCHECK(parsed_payload);
445 const auto& vp9_header = absl::get<RTPVideoHeaderVP9>(
446 parsed_payload->video_header.video_type_header);
447 int temporal_idx = vp9_header.temporal_idx;
448 int spatial_idx = vp9_header.spatial_idx;
449 return (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
450 temporal_idx <= selected_tl_) &&
451 (selected_sl_ < 0 || spatial_idx == kNoSpatialIdx ||
452 spatial_idx <= selected_sl_);
453 }
454
455 return true;
456 }
457
PollStats()458 void VideoAnalyzer::PollStats() {
459 MutexLock lock(&comparison_lock_);
460
461 Call::Stats call_stats = call_->GetStats();
462 send_bandwidth_bps_.AddSample(call_stats.send_bandwidth_bps);
463
464 VideoSendStream::Stats send_stats = send_stream_->GetStats();
465 // It's not certain that we yet have estimates for any of these stats.
466 // Check that they are positive before mixing them in.
467 if (send_stats.encode_frame_rate > 0)
468 encode_frame_rate_.AddSample(send_stats.encode_frame_rate);
469 if (send_stats.avg_encode_time_ms > 0)
470 encode_time_ms_.AddSample(send_stats.avg_encode_time_ms);
471 if (send_stats.encode_usage_percent > 0)
472 encode_usage_percent_.AddSample(send_stats.encode_usage_percent);
473 if (send_stats.media_bitrate_bps > 0)
474 media_bitrate_bps_.AddSample(send_stats.media_bitrate_bps);
475 size_t fec_bytes = 0;
476 for (const auto& kv : send_stats.substreams) {
477 fec_bytes += kv.second.rtp_stats.fec.payload_bytes +
478 kv.second.rtp_stats.fec.padding_bytes;
479 }
480 fec_bitrate_bps_.AddSample((fec_bytes - last_fec_bytes_) * 8);
481 last_fec_bytes_ = fec_bytes;
482
483 if (receive_stream_ != nullptr) {
484 VideoReceiveStream::Stats receive_stats = receive_stream_->GetStats();
485 // |total_decode_time_ms| gives a good estimate of the mean decode time,
486 // |decode_ms| is used to keep track of the standard deviation.
487 if (receive_stats.frames_decoded > 0)
488 mean_decode_time_ms_ =
489 static_cast<double>(receive_stats.total_decode_time_ms) /
490 receive_stats.frames_decoded;
491 if (receive_stats.decode_ms > 0)
492 decode_time_ms_.AddSample(receive_stats.decode_ms);
493 if (receive_stats.max_decode_ms > 0)
494 decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
495 if (receive_stats.width > 0 && receive_stats.height > 0) {
496 pixels_.AddSample(receive_stats.width * receive_stats.height);
497 }
498
499 // |frames_decoded| and |frames_rendered| are used because they are more
500 // accurate than |decode_frame_rate| and |render_frame_rate|.
501 // The latter two are calculated on a momentary basis.
502 const double total_frames_duration_sec_double =
503 static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
504 if (total_frames_duration_sec_double > 0) {
505 decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
506 total_frames_duration_sec_double;
507 render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
508 total_frames_duration_sec_double;
509 }
510
511 // Freeze metrics.
512 freeze_count_ = receive_stats.freeze_count;
513 total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
514 total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
515 sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
516 }
517
518 if (audio_receive_stream_ != nullptr) {
519 AudioReceiveStream::Stats receive_stats = audio_receive_stream_->GetStats();
520 audio_expand_rate_.AddSample(receive_stats.expand_rate);
521 audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate);
522 audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms);
523 }
524
525 memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
526 }
527
FrameComparisonThread(void * obj)528 void VideoAnalyzer::FrameComparisonThread(void* obj) {
529 VideoAnalyzer* analyzer = static_cast<VideoAnalyzer*>(obj);
530 while (analyzer->CompareFrames()) {
531 }
532 }
533
CompareFrames()534 bool VideoAnalyzer::CompareFrames() {
535 if (AllFramesRecorded())
536 return false;
537
538 FrameComparison comparison;
539
540 if (!PopComparison(&comparison)) {
541 // Wait until new comparison task is available, or test is done.
542 // If done, wake up remaining threads waiting.
543 comparison_available_event_.Wait(1000);
544 if (AllFramesRecorded()) {
545 comparison_available_event_.Set();
546 return false;
547 }
548 return true; // Try again.
549 }
550
551 StartExcludingCpuThreadTime();
552
553 PerformFrameComparison(comparison);
554
555 StopExcludingCpuThreadTime();
556
557 if (FrameProcessed()) {
558 done_.Set();
559 comparison_available_event_.Set();
560 return false;
561 }
562
563 return true;
564 }
565
PopComparison(VideoAnalyzer::FrameComparison * comparison)566 bool VideoAnalyzer::PopComparison(VideoAnalyzer::FrameComparison* comparison) {
567 MutexLock lock(&comparison_lock_);
568 // If AllFramesRecorded() is true, it means we have already popped
569 // frames_to_process_ frames from comparisons_, so there is no more work
570 // for this thread to be done. frames_processed_ might still be lower if
571 // all comparisons are not done, but those frames are currently being
572 // worked on by other threads.
573 if (comparisons_.empty() || AllFramesRecordedLocked())
574 return false;
575
576 *comparison = comparisons_.front();
577 comparisons_.pop_front();
578
579 FrameRecorded();
580 return true;
581 }
582
FrameRecorded()583 void VideoAnalyzer::FrameRecorded() {
584 ++frames_recorded_;
585 }
586
AllFramesRecorded()587 bool VideoAnalyzer::AllFramesRecorded() {
588 MutexLock lock(&comparison_lock_);
589 return AllFramesRecordedLocked();
590 }
591
AllFramesRecordedLocked()592 bool VideoAnalyzer::AllFramesRecordedLocked() {
593 RTC_DCHECK(frames_recorded_ <= frames_to_process_);
594 return frames_recorded_ == frames_to_process_ ||
595 (clock_->CurrentTime() > test_end_ && comparisons_.empty()) || quit_;
596 }
597
FrameProcessed()598 bool VideoAnalyzer::FrameProcessed() {
599 MutexLock lock(&comparison_lock_);
600 ++frames_processed_;
601 assert(frames_processed_ <= frames_to_process_);
602 return frames_processed_ == frames_to_process_ ||
603 (clock_->CurrentTime() > test_end_ && comparisons_.empty());
604 }
605
PrintResults()606 void VideoAnalyzer::PrintResults() {
607 using ::webrtc::test::ImproveDirection;
608
609 StopMeasuringCpuProcessTime();
610 int dropped_frames_diff;
611 {
612 MutexLock lock(&lock_);
613 dropped_frames_diff = dropped_frames_before_first_encode_ +
614 dropped_frames_before_rendering_ + frames_.size();
615 }
616 MutexLock lock(&comparison_lock_);
617 PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter);
618 PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter);
619 PrintResult("sender_time", sender_time_, "ms",
620 ImproveDirection::kSmallerIsBetter);
621 PrintResult("receiver_time", receiver_time_, "ms",
622 ImproveDirection::kSmallerIsBetter);
623 PrintResult("network_time", network_time_, "ms",
624 ImproveDirection::kSmallerIsBetter);
625 PrintResult("total_delay_incl_network", end_to_end_, "ms",
626 ImproveDirection::kSmallerIsBetter);
627 PrintResult("time_between_rendered_frames", rendered_delta_, "ms",
628 ImproveDirection::kSmallerIsBetter);
629 PrintResult("encode_frame_rate", encode_frame_rate_, "fps",
630 ImproveDirection::kBiggerIsBetter);
631 PrintResult("encode_time", encode_time_ms_, "ms",
632 ImproveDirection::kSmallerIsBetter);
633 PrintResult("media_bitrate", media_bitrate_bps_, "bps",
634 ImproveDirection::kNone);
635 PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone);
636 PrintResult("send_bandwidth", send_bandwidth_bps_, "bps",
637 ImproveDirection::kNone);
638 PrintResult("pixels_per_frame", pixels_, "count",
639 ImproveDirection::kBiggerIsBetter);
640
641 test::PrintResult("decode_frame_rate", "", test_label_.c_str(),
642 decode_frame_rate_, "fps", false,
643 ImproveDirection::kBiggerIsBetter);
644 test::PrintResult("render_frame_rate", "", test_label_.c_str(),
645 render_frame_rate_, "fps", false,
646 ImproveDirection::kBiggerIsBetter);
647
648 // Record the time from the last freeze until the last rendered frame to
649 // ensure we cover the full timespan of the session. Otherwise the metric
650 // would penalize an early freeze followed by no freezes until the end.
651 time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_);
652
653 // Freeze metrics.
654 PrintResult("time_between_freezes", time_between_freezes_, "ms",
655 ImproveDirection::kBiggerIsBetter);
656
657 const double freeze_count_double = static_cast<double>(freeze_count_);
658 const double total_freezes_duration_ms_double =
659 static_cast<double>(total_freezes_duration_ms_);
660 const double total_frames_duration_ms_double =
661 static_cast<double>(total_frames_duration_ms_);
662
663 if (total_frames_duration_ms_double > 0) {
664 test::PrintResult(
665 "freeze_duration_ratio", "", test_label_.c_str(),
666 total_freezes_duration_ms_double / total_frames_duration_ms_double,
667 "unitless", false, ImproveDirection::kSmallerIsBetter);
668 RTC_DCHECK_LE(total_freezes_duration_ms_double,
669 total_frames_duration_ms_double);
670
671 constexpr double ms_per_minute = 60 * 1000;
672 const double total_frames_duration_min =
673 total_frames_duration_ms_double / ms_per_minute;
674 if (total_frames_duration_min > 0) {
675 test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(),
676 freeze_count_double / total_frames_duration_min,
677 "unitless", false, ImproveDirection::kSmallerIsBetter);
678 }
679 }
680
681 test::PrintResult("freeze_duration_average", "", test_label_.c_str(),
682 freeze_count_double > 0
683 ? total_freezes_duration_ms_double / freeze_count_double
684 : 0,
685 "ms", false, ImproveDirection::kSmallerIsBetter);
686
687 if (1000 * sum_squared_frame_durations_ > 0) {
688 test::PrintResult(
689 "harmonic_frame_rate", "", test_label_.c_str(),
690 total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
691 "fps", false, ImproveDirection::kBiggerIsBetter);
692 }
693
694 if (worst_frame_) {
695 test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr,
696 "dB", false, ImproveDirection::kBiggerIsBetter);
697 }
698
699 if (receive_stream_ != nullptr) {
700 PrintResultWithExternalMean("decode_time", mean_decode_time_ms_,
701 decode_time_ms_, "ms",
702 ImproveDirection::kSmallerIsBetter);
703 }
704 dropped_frames_ += dropped_frames_diff;
705 test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_,
706 "count", false, ImproveDirection::kSmallerIsBetter);
707 test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
708 "%", false, ImproveDirection::kSmallerIsBetter);
709
710 #if defined(WEBRTC_WIN)
711 // On Linux and Mac in Resident Set some unused pages may be counted.
712 // Therefore this metric will depend on order in which tests are run and
713 // will be flaky.
714 PrintResult("memory_usage", memory_usage_, "sizeInBytes",
715 ImproveDirection::kSmallerIsBetter);
716 #endif
717
718 // Saving only the worst frame for manual analysis. Intention here is to
719 // only detect video corruptions and not to track picture quality. Thus,
720 // jpeg is used here.
721 if (absl::GetFlag(FLAGS_save_worst_frame) && worst_frame_) {
722 std::string output_dir;
723 test::GetTestArtifactsDir(&output_dir);
724 std::string output_path =
725 test::JoinFilename(output_dir, test_label_ + ".jpg");
726 RTC_LOG(LS_INFO) << "Saving worst frame to " << output_path;
727 test::JpegFrameWriter frame_writer(output_path);
728 RTC_CHECK(
729 frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/));
730 }
731
732 if (audio_receive_stream_ != nullptr) {
733 PrintResult("audio_expand_rate", audio_expand_rate_, "unitless",
734 ImproveDirection::kSmallerIsBetter);
735 PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless",
736 ImproveDirection::kSmallerIsBetter);
737 PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms",
738 ImproveDirection::kNone);
739 }
740
741 // Disable quality check for quick test, as quality checks may fail
742 // because too few samples were collected.
743 if (!is_quick_test_enabled_) {
744 EXPECT_GT(*psnr_.GetMean(), avg_psnr_threshold_);
745 EXPECT_GT(*ssim_.GetMean(), avg_ssim_threshold_);
746 }
747 }
748
PerformFrameComparison(const VideoAnalyzer::FrameComparison & comparison)749 void VideoAnalyzer::PerformFrameComparison(
750 const VideoAnalyzer::FrameComparison& comparison) {
751 // Perform expensive psnr and ssim calculations while not holding lock.
752 double psnr = -1.0;
753 double ssim = -1.0;
754 if (comparison.reference && !comparison.dropped) {
755 psnr = I420PSNR(&*comparison.reference, &*comparison.render);
756 ssim = I420SSIM(&*comparison.reference, &*comparison.render);
757 }
758
759 MutexLock lock(&comparison_lock_);
760
761 if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) {
762 worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render});
763 }
764
765 if (graph_data_output_file_) {
766 samples_.push_back(Sample(comparison.dropped, comparison.input_time_ms,
767 comparison.send_time_ms, comparison.recv_time_ms,
768 comparison.render_time_ms,
769 comparison.encoded_frame_size, psnr, ssim));
770 }
771 if (psnr >= 0.0)
772 psnr_.AddSample(psnr);
773 if (ssim >= 0.0)
774 ssim_.AddSample(ssim);
775
776 if (comparison.dropped) {
777 ++dropped_frames_;
778 return;
779 }
780 if (last_unfreeze_time_ms_ == 0)
781 last_unfreeze_time_ms_ = comparison.render_time_ms;
782 if (last_render_time_ != 0) {
783 const int64_t render_delta_ms =
784 comparison.render_time_ms - last_render_time_;
785 rendered_delta_.AddSample(render_delta_ms);
786 if (last_render_delta_ms_ != 0 &&
787 render_delta_ms - last_render_delta_ms_ > 150) {
788 time_between_freezes_.AddSample(last_render_time_ -
789 last_unfreeze_time_ms_);
790 last_unfreeze_time_ms_ = comparison.render_time_ms;
791 }
792 last_render_delta_ms_ = render_delta_ms;
793 }
794 last_render_time_ = comparison.render_time_ms;
795
796 sender_time_.AddSample(comparison.send_time_ms - comparison.input_time_ms);
797 if (comparison.recv_time_ms > 0) {
798 // If recv_time_ms == 0, this frame consisted of a packets which were all
799 // lost in the transport. Since we were able to render the frame, however,
800 // the dropped packets were recovered by FlexFEC. The FlexFEC recovery
801 // happens internally in Call, and we can therefore here not know which
802 // FEC packets that protected the lost media packets. Consequently, we
803 // were not able to record a meaningful recv_time_ms. We therefore skip
804 // this sample.
805 //
806 // The reasoning above does not hold for ULPFEC and RTX, as for those
807 // strategies the timestamp of the received packets is set to the
808 // timestamp of the protected/retransmitted media packet. I.e., then
809 // recv_time_ms != 0, even though the media packets were lost.
810 receiver_time_.AddSample(comparison.render_time_ms -
811 comparison.recv_time_ms);
812 network_time_.AddSample(comparison.recv_time_ms - comparison.send_time_ms);
813 }
814 end_to_end_.AddSample(comparison.render_time_ms - comparison.input_time_ms);
815 encoded_frame_size_.AddSample(comparison.encoded_frame_size);
816 }
817
PrintResult(const char * result_type,Statistics stats,const char * unit,webrtc::test::ImproveDirection improve_direction)818 void VideoAnalyzer::PrintResult(
819 const char* result_type,
820 Statistics stats,
821 const char* unit,
822 webrtc::test::ImproveDirection improve_direction) {
823 test::PrintResultMeanAndError(
824 result_type, "", test_label_.c_str(), stats.GetMean().value_or(0),
825 stats.GetStandardDeviation().value_or(0), unit, false, improve_direction);
826 }
827
PrintResultWithExternalMean(const char * result_type,double mean,Statistics stats,const char * unit,webrtc::test::ImproveDirection improve_direction)828 void VideoAnalyzer::PrintResultWithExternalMean(
829 const char* result_type,
830 double mean,
831 Statistics stats,
832 const char* unit,
833 webrtc::test::ImproveDirection improve_direction) {
834 // If the true mean is different than the sample mean, the sample variance is
835 // too low. The sample variance given a known mean is obtained by adding the
836 // squared error between the true mean and the sample mean.
837 double compensated_variance =
838 stats.Size() > 0
839 ? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0)
840 : 0.0;
841 test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean,
842 std::sqrt(compensated_variance), unit, false,
843 improve_direction);
844 }
845
PrintSamplesToFile()846 void VideoAnalyzer::PrintSamplesToFile() {
847 FILE* out = graph_data_output_file_;
848 MutexLock lock(&comparison_lock_);
849 absl::c_sort(samples_, [](const Sample& A, const Sample& B) -> bool {
850 return A.input_time_ms < B.input_time_ms;
851 });
852
853 fprintf(out, "%s\n", graph_title_.c_str());
854 fprintf(out, "%" RTC_PRIuS "\n", samples_.size());
855 fprintf(out,
856 "dropped "
857 "input_time_ms "
858 "send_time_ms "
859 "recv_time_ms "
860 "render_time_ms "
861 "encoded_frame_size "
862 "psnr "
863 "ssim "
864 "encode_time_ms\n");
865 for (const Sample& sample : samples_) {
866 fprintf(out,
867 "%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" RTC_PRIuS
868 " %lf %lf\n",
869 sample.dropped, sample.input_time_ms, sample.send_time_ms,
870 sample.recv_time_ms, sample.render_time_ms,
871 sample.encoded_frame_size, sample.psnr, sample.ssim);
872 }
873 }
874
AddCapturedFrameForComparison(const VideoFrame & video_frame)875 void VideoAnalyzer::AddCapturedFrameForComparison(
876 const VideoFrame& video_frame) {
877 bool must_capture = false;
878 {
879 MutexLock lock(&comparison_lock_);
880 must_capture = captured_frames_ < frames_to_process_;
881 if (must_capture) {
882 ++captured_frames_;
883 }
884 }
885 if (must_capture) {
886 MutexLock lock(&lock_);
887 frames_.push_back(video_frame);
888 }
889 }
890
AddFrameComparison(const VideoFrame & reference,const VideoFrame & render,bool dropped,int64_t render_time_ms)891 void VideoAnalyzer::AddFrameComparison(const VideoFrame& reference,
892 const VideoFrame& render,
893 bool dropped,
894 int64_t render_time_ms) {
895 int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp());
896 int64_t send_time_ms = send_times_[reference_timestamp];
897 send_times_.erase(reference_timestamp);
898 int64_t recv_time_ms = recv_times_[reference_timestamp];
899 recv_times_.erase(reference_timestamp);
900
901 // TODO(ivica): Make this work for > 2 streams.
902 auto it = encoded_frame_sizes_.find(reference_timestamp);
903 if (it == encoded_frame_sizes_.end())
904 it = encoded_frame_sizes_.find(reference_timestamp - 1);
905 size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second;
906 if (it != encoded_frame_sizes_.end())
907 encoded_frame_sizes_.erase(it);
908
909 MutexLock lock(&comparison_lock_);
910 if (comparisons_.size() < kMaxComparisons) {
911 comparisons_.push_back(FrameComparison(
912 reference, render, dropped, reference.ntp_time_ms(), send_time_ms,
913 recv_time_ms, render_time_ms, encoded_size));
914 } else {
915 comparisons_.push_back(FrameComparison(dropped, reference.ntp_time_ms(),
916 send_time_ms, recv_time_ms,
917 render_time_ms, encoded_size));
918 }
919 comparison_available_event_.Set();
920 }
921
FrameComparison()922 VideoAnalyzer::FrameComparison::FrameComparison()
923 : dropped(false),
924 input_time_ms(0),
925 send_time_ms(0),
926 recv_time_ms(0),
927 render_time_ms(0),
928 encoded_frame_size(0) {}
929
FrameComparison(const VideoFrame & reference,const VideoFrame & render,bool dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size)930 VideoAnalyzer::FrameComparison::FrameComparison(const VideoFrame& reference,
931 const VideoFrame& render,
932 bool dropped,
933 int64_t input_time_ms,
934 int64_t send_time_ms,
935 int64_t recv_time_ms,
936 int64_t render_time_ms,
937 size_t encoded_frame_size)
938 : reference(reference),
939 render(render),
940 dropped(dropped),
941 input_time_ms(input_time_ms),
942 send_time_ms(send_time_ms),
943 recv_time_ms(recv_time_ms),
944 render_time_ms(render_time_ms),
945 encoded_frame_size(encoded_frame_size) {}
946
FrameComparison(bool dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size)947 VideoAnalyzer::FrameComparison::FrameComparison(bool dropped,
948 int64_t input_time_ms,
949 int64_t send_time_ms,
950 int64_t recv_time_ms,
951 int64_t render_time_ms,
952 size_t encoded_frame_size)
953 : dropped(dropped),
954 input_time_ms(input_time_ms),
955 send_time_ms(send_time_ms),
956 recv_time_ms(recv_time_ms),
957 render_time_ms(render_time_ms),
958 encoded_frame_size(encoded_frame_size) {}
959
Sample(int dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size,double psnr,double ssim)960 VideoAnalyzer::Sample::Sample(int dropped,
961 int64_t input_time_ms,
962 int64_t send_time_ms,
963 int64_t recv_time_ms,
964 int64_t render_time_ms,
965 size_t encoded_frame_size,
966 double psnr,
967 double ssim)
968 : dropped(dropped),
969 input_time_ms(input_time_ms),
970 send_time_ms(send_time_ms),
971 recv_time_ms(recv_time_ms),
972 render_time_ms(render_time_ms),
973 encoded_frame_size(encoded_frame_size),
974 psnr(psnr),
975 ssim(ssim) {}
976
CapturedFrameForwarder(VideoAnalyzer * analyzer,Clock * clock,int frames_to_capture,TimeDelta test_duration)977 VideoAnalyzer::CapturedFrameForwarder::CapturedFrameForwarder(
978 VideoAnalyzer* analyzer,
979 Clock* clock,
980 int frames_to_capture,
981 TimeDelta test_duration)
982 : analyzer_(analyzer),
983 send_stream_input_(nullptr),
984 video_source_(nullptr),
985 clock_(clock),
986 captured_frames_(0),
987 frames_to_capture_(frames_to_capture),
988 test_end_(clock->CurrentTime() + test_duration) {}
989
SetSource(VideoSourceInterface<VideoFrame> * video_source)990 void VideoAnalyzer::CapturedFrameForwarder::SetSource(
991 VideoSourceInterface<VideoFrame>* video_source) {
992 video_source_ = video_source;
993 }
994
OnFrame(const VideoFrame & video_frame)995 void VideoAnalyzer::CapturedFrameForwarder::OnFrame(
996 const VideoFrame& video_frame) {
997 VideoFrame copy = video_frame;
998 // Frames from the capturer does not have a rtp timestamp.
999 // Create one so it can be used for comparison.
1000 RTC_DCHECK_EQ(0, video_frame.timestamp());
1001 if (video_frame.ntp_time_ms() == 0)
1002 copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
1003 copy.set_timestamp(copy.ntp_time_ms() * 90);
1004 analyzer_->AddCapturedFrameForComparison(copy);
1005 MutexLock lock(&lock_);
1006 ++captured_frames_;
1007 if (send_stream_input_ && clock_->CurrentTime() <= test_end_ &&
1008 captured_frames_ <= frames_to_capture_) {
1009 send_stream_input_->OnFrame(copy);
1010 }
1011 }
1012
AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame> * sink,const rtc::VideoSinkWants & wants)1013 void VideoAnalyzer::CapturedFrameForwarder::AddOrUpdateSink(
1014 rtc::VideoSinkInterface<VideoFrame>* sink,
1015 const rtc::VideoSinkWants& wants) {
1016 {
1017 MutexLock lock(&lock_);
1018 RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink);
1019 send_stream_input_ = sink;
1020 }
1021 if (video_source_) {
1022 video_source_->AddOrUpdateSink(this, wants);
1023 }
1024 }
1025
RemoveSink(rtc::VideoSinkInterface<VideoFrame> * sink)1026 void VideoAnalyzer::CapturedFrameForwarder::RemoveSink(
1027 rtc::VideoSinkInterface<VideoFrame>* sink) {
1028 MutexLock lock(&lock_);
1029 RTC_DCHECK(sink == send_stream_input_);
1030 send_stream_input_ = nullptr;
1031 }
1032
1033 } // namespace webrtc
1034