1 /* 2 * Copyright 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #ifndef VIDEO_VIDEO_ANALYZER_H_ 11 #define VIDEO_VIDEO_ANALYZER_H_ 12 13 #include <deque> 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "api/task_queue/task_queue_base.h" 20 #include "api/video/video_source_interface.h" 21 #include "modules/rtp_rtcp/source/rtp_packet.h" 22 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" 23 #include "rtc_base/event.h" 24 #include "rtc_base/numerics/running_statistics.h" 25 #include "rtc_base/platform_thread.h" 26 #include "rtc_base/synchronization/mutex.h" 27 #include "rtc_base/time_utils.h" 28 #include "test/layer_filtering_transport.h" 29 #include "test/rtp_file_writer.h" 30 #include "test/testsupport/perf_test.h" 31 32 namespace webrtc { 33 34 class VideoAnalyzer : public PacketReceiver, 35 public Transport, 36 public rtc::VideoSinkInterface<VideoFrame> { 37 public: 38 using Statistics = RunningStatistics<double>; 39 40 VideoAnalyzer(test::LayerFilteringTransport* transport, 41 const std::string& test_label, 42 double avg_psnr_threshold, 43 double avg_ssim_threshold, 44 int duration_frames, 45 TimeDelta test_duration, 46 FILE* graph_data_output_file, 47 const std::string& graph_title, 48 uint32_t ssrc_to_analyze, 49 uint32_t rtx_ssrc_to_analyze, 50 size_t selected_stream, 51 int selected_sl, 52 int selected_tl, 53 bool is_quick_test_enabled, 54 Clock* clock, 55 std::string rtp_dump_name, 56 TaskQueueBase* task_queue); 57 ~VideoAnalyzer(); 58 59 virtual void SetReceiver(PacketReceiver* receiver); 60 void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source, 61 bool respect_sink_wants); 62 void SetCall(Call* call); 63 void SetSendStream(VideoSendStream* stream); 64 void SetReceiveStream(VideoReceiveStream* stream); 65 void SetAudioReceiveStream(AudioReceiveStream* recv_stream); 66 67 rtc::VideoSinkInterface<VideoFrame>* InputInterface(); 68 rtc::VideoSourceInterface<VideoFrame>* OutputInterface(); 69 70 DeliveryStatus DeliverPacket(MediaType media_type, 71 rtc::CopyOnWriteBuffer packet, 72 int64_t packet_time_us) override; 73 74 void PreEncodeOnFrame(const VideoFrame& video_frame); 75 void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp); 76 77 bool SendRtp(const uint8_t* packet, 78 size_t length, 79 const PacketOptions& options) override; 80 81 bool SendRtcp(const uint8_t* packet, size_t length) override; 82 void OnFrame(const VideoFrame& video_frame) override; 83 void Wait(); 84 85 void StartMeasuringCpuProcessTime(); 86 void StopMeasuringCpuProcessTime(); 87 void StartExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_); 88 void StopExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_); 89 double GetCpuUsagePercent() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_); 90 91 test::LayerFilteringTransport* const transport_; 92 PacketReceiver* receiver_; 93 94 private: 95 struct FrameComparison { 96 FrameComparison(); 97 FrameComparison(const VideoFrame& reference, 98 const VideoFrame& render, 99 bool dropped, 100 int64_t input_time_ms, 101 int64_t send_time_ms, 102 int64_t recv_time_ms, 103 int64_t render_time_ms, 104 size_t encoded_frame_size); 105 FrameComparison(bool dropped, 106 int64_t input_time_ms, 107 int64_t send_time_ms, 108 int64_t recv_time_ms, 109 int64_t render_time_ms, 110 size_t encoded_frame_size); 111 112 absl::optional<VideoFrame> reference; 113 absl::optional<VideoFrame> render; 114 bool dropped; 115 int64_t input_time_ms; 116 int64_t send_time_ms; 117 int64_t recv_time_ms; 118 int64_t render_time_ms; 119 size_t encoded_frame_size; 120 }; 121 122 struct Sample { 123 Sample(int dropped, 124 int64_t input_time_ms, 125 int64_t send_time_ms, 126 int64_t recv_time_ms, 127 int64_t render_time_ms, 128 size_t encoded_frame_size, 129 double psnr, 130 double ssim); 131 132 int dropped; 133 int64_t input_time_ms; 134 int64_t send_time_ms; 135 int64_t recv_time_ms; 136 int64_t render_time_ms; 137 size_t encoded_frame_size; 138 double psnr; 139 double ssim; 140 }; 141 142 // Implements VideoSinkInterface to receive captured frames from a 143 // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act 144 // as a source to VideoSendStream. 145 // It forwards all input frames to the VideoAnalyzer for later comparison and 146 // forwards the captured frames to the VideoSendStream. 147 class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>, 148 public rtc::VideoSourceInterface<VideoFrame> { 149 public: 150 CapturedFrameForwarder(VideoAnalyzer* analyzer, 151 Clock* clock, 152 int frames_to_capture, 153 TimeDelta test_duration); 154 void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source); 155 156 private: 157 void OnFrame(const VideoFrame& video_frame) 158 RTC_LOCKS_EXCLUDED(lock_) override; 159 160 // Called when |send_stream_.SetSource()| is called. 161 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink, 162 const rtc::VideoSinkWants& wants) 163 RTC_LOCKS_EXCLUDED(lock_) override; 164 165 // Called by |send_stream_| when |send_stream_.SetSource()| is called. 166 void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) 167 RTC_LOCKS_EXCLUDED(lock_) override; 168 169 VideoAnalyzer* const analyzer_; 170 Mutex lock_; 171 rtc::VideoSinkInterface<VideoFrame>* send_stream_input_ 172 RTC_GUARDED_BY(lock_); 173 VideoSourceInterface<VideoFrame>* video_source_; 174 Clock* clock_; 175 int captured_frames_ RTC_GUARDED_BY(lock_); 176 const int frames_to_capture_; 177 const Timestamp test_end_; 178 }; 179 180 struct FrameWithPsnr { 181 double psnr; 182 VideoFrame frame; 183 }; 184 185 bool IsInSelectedSpatialAndTemporalLayer(const RtpPacket& rtp_packet); 186 187 void AddFrameComparison(const VideoFrame& reference, 188 const VideoFrame& render, 189 bool dropped, 190 int64_t render_time_ms) 191 RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); 192 193 void PollStats() RTC_LOCKS_EXCLUDED(comparison_lock_); 194 static void FrameComparisonThread(void* obj); 195 bool CompareFrames(); 196 bool PopComparison(FrameComparison* comparison); 197 // Increment counter for number of frames received for comparison. 198 void FrameRecorded() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_); 199 // Returns true if all frames to be compared have been taken from the queue. 200 bool AllFramesRecorded() RTC_LOCKS_EXCLUDED(comparison_lock_); 201 bool AllFramesRecordedLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_); 202 // Increase count of number of frames processed. Returns true if this was the 203 // last frame to be processed. 204 bool FrameProcessed() RTC_LOCKS_EXCLUDED(comparison_lock_); 205 void PrintResults() RTC_LOCKS_EXCLUDED(lock_, comparison_lock_); 206 void PerformFrameComparison(const FrameComparison& comparison) 207 RTC_LOCKS_EXCLUDED(comparison_lock_); 208 void PrintResult(const char* result_type, 209 Statistics stats, 210 const char* unit, 211 webrtc::test::ImproveDirection improve_direction); 212 void PrintResultWithExternalMean( 213 const char* result_type, 214 double mean, 215 Statistics stats, 216 const char* unit, 217 webrtc::test::ImproveDirection improve_direction); 218 void PrintSamplesToFile(void) RTC_LOCKS_EXCLUDED(comparison_lock_); 219 void AddCapturedFrameForComparison(const VideoFrame& video_frame) 220 RTC_LOCKS_EXCLUDED(lock_, comparison_lock_); 221 222 Call* call_; 223 VideoSendStream* send_stream_; 224 VideoReceiveStream* receive_stream_; 225 AudioReceiveStream* audio_receive_stream_; 226 CapturedFrameForwarder captured_frame_forwarder_; 227 const std::string test_label_; 228 FILE* const graph_data_output_file_; 229 const std::string graph_title_; 230 const uint32_t ssrc_to_analyze_; 231 const uint32_t rtx_ssrc_to_analyze_; 232 const size_t selected_stream_; 233 const int selected_sl_; 234 const int selected_tl_; 235 236 Mutex comparison_lock_; 237 std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_); 238 Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_); 239 Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_); 240 Statistics network_time_ RTC_GUARDED_BY(comparison_lock_); 241 Statistics psnr_ RTC_GUARDED_BY(comparison_lock_); 242 Statistics ssim_ RTC_GUARDED_BY(comparison_lock_); 243 Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_); 244 Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_); 245 Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_); 246 Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); 247 Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_); 248 Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_); 249 double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); 250 Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); 251 Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_); 252 Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); 253 Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); 254 Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); 255 Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_); 256 Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_); 257 Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_); 258 Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_); 259 Statistics pixels_ RTC_GUARDED_BY(comparison_lock_); 260 // Rendered frame with worst PSNR is saved for further analysis. 261 absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_); 262 // Freeze metrics. 263 Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); 264 uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_); 265 uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_); 266 uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_); 267 double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_); 268 269 double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); 270 double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_); 271 272 size_t last_fec_bytes_; 273 274 Mutex lock_ RTC_ACQUIRED_BEFORE(comparison_lock_) 275 RTC_ACQUIRED_BEFORE(cpu_measurement_lock_); 276 const int frames_to_process_; 277 const Timestamp test_end_; 278 int frames_recorded_ RTC_GUARDED_BY(comparison_lock_); 279 int frames_processed_ RTC_GUARDED_BY(comparison_lock_); 280 int captured_frames_ RTC_GUARDED_BY(comparison_lock_); 281 int dropped_frames_ RTC_GUARDED_BY(comparison_lock_); 282 int dropped_frames_before_first_encode_ RTC_GUARDED_BY(lock_); 283 int dropped_frames_before_rendering_ RTC_GUARDED_BY(lock_); 284 int64_t last_render_time_ RTC_GUARDED_BY(comparison_lock_); 285 int64_t last_render_delta_ms_ RTC_GUARDED_BY(comparison_lock_); 286 int64_t last_unfreeze_time_ms_ RTC_GUARDED_BY(comparison_lock_); 287 uint32_t rtp_timestamp_delta_ RTC_GUARDED_BY(lock_); 288 289 Mutex cpu_measurement_lock_; 290 int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_); 291 int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_); 292 293 std::deque<VideoFrame> frames_ RTC_GUARDED_BY(lock_); 294 absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(lock_); 295 rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(lock_); 296 std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(lock_); 297 std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(lock_); 298 std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(lock_); 299 absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(lock_); 300 absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(lock_); 301 const double avg_psnr_threshold_; 302 const double avg_ssim_threshold_; 303 bool is_quick_test_enabled_; 304 305 std::vector<rtc::PlatformThread*> comparison_thread_pool_; 306 rtc::Event comparison_available_event_; 307 std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_); 308 bool quit_ RTC_GUARDED_BY(comparison_lock_); 309 rtc::Event done_; 310 311 std::unique_ptr<VideoRtpDepacketizer> vp8_depacketizer_; 312 std::unique_ptr<VideoRtpDepacketizer> vp9_depacketizer_; 313 std::unique_ptr<test::RtpFileWriter> rtp_file_writer_; 314 Clock* const clock_; 315 const int64_t start_ms_; 316 TaskQueueBase* task_queue_; 317 }; 318 319 } // namespace webrtc 320 #endif // VIDEO_VIDEO_ANALYZER_H_ 321