1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 12 #define VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/task_queue/task_queue_base.h" 21 #include "api/units/timestamp.h" 22 #include "call/video_receive_stream.h" 23 #include "modules/include/module_common_types.h" 24 #include "modules/video_coding/include/video_coding_defines.h" 25 #include "rtc_base/numerics/histogram_percentile_counter.h" 26 #include "rtc_base/numerics/moving_max_counter.h" 27 #include "rtc_base/numerics/sample_counter.h" 28 #include "rtc_base/rate_statistics.h" 29 #include "rtc_base/rate_tracker.h" 30 #include "rtc_base/synchronization/sequence_checker.h" 31 #include "rtc_base/task_utils/pending_task_safety_flag.h" 32 #include "rtc_base/thread_annotations.h" 33 #include "rtc_base/thread_checker.h" 34 #include "video/quality_threshold.h" 35 #include "video/stats_counter.h" 36 #include "video/video_quality_observer2.h" 37 38 namespace webrtc { 39 40 class Clock; 41 struct CodecSpecificInfo; 42 43 namespace internal { 44 // Declared in video_receive_stream2.h. 45 struct VideoFrameMetaData; 46 47 class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, 48 public RtcpCnameCallback, 49 public RtcpPacketTypeCounterObserver { 50 public: 51 ReceiveStatisticsProxy(const VideoReceiveStream::Config* config, 52 Clock* clock, 53 TaskQueueBase* worker_thread); 54 ~ReceiveStatisticsProxy() override; 55 56 VideoReceiveStream::Stats GetStats() const; 57 58 void OnDecodedFrame(const VideoFrame& frame, 59 absl::optional<uint8_t> qp, 60 int32_t decode_time_ms, 61 VideoContentType content_type); 62 63 // Called asyncronously on the worker thread as a result of a call to the 64 // above OnDecodedFrame method, which is called back on the thread where 65 // the actual decoding happens. 66 void OnDecodedFrame(const VideoFrameMetaData& frame_meta, 67 absl::optional<uint8_t> qp, 68 int32_t decode_time_ms, 69 VideoContentType content_type); 70 71 void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, 72 int64_t sync_offset_ms, 73 double estimated_freq_khz); 74 void OnRenderedFrame(const VideoFrameMetaData& frame_meta); 75 void OnIncomingPayloadType(int payload_type); 76 void OnDecoderImplementationName(const char* implementation_name); 77 78 void OnPreDecode(VideoCodecType codec_type, int qp); 79 80 void OnUniqueFramesCounted(int num_unique_frames); 81 82 // Indicates video stream has been paused (no incoming packets). 83 void OnStreamInactive(); 84 85 // Overrides VCMReceiveStatisticsCallback. 86 void OnCompleteFrame(bool is_keyframe, 87 size_t size_bytes, 88 VideoContentType content_type) override; 89 void OnDroppedFrames(uint32_t frames_dropped) override; 90 void OnFrameBufferTimingsUpdated(int max_decode_ms, 91 int current_delay_ms, 92 int target_delay_ms, 93 int jitter_buffer_ms, 94 int min_playout_delay_ms, 95 int render_delay_ms) override; 96 97 void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override; 98 99 // Overrides RtcpCnameCallback. 100 void OnCname(uint32_t ssrc, absl::string_view cname) override; 101 102 // Overrides RtcpPacketTypeCounterObserver. 103 void RtcpPacketTypesCounterUpdated( 104 uint32_t ssrc, 105 const RtcpPacketTypeCounter& packet_counter) override; 106 107 void OnRttUpdate(int64_t avg_rtt_ms); 108 109 // Notification methods that are used to check our internal state and validate 110 // threading assumptions. These are called by VideoReceiveStream. 111 void DecoderThreadStarting(); 112 void DecoderThreadStopped(); 113 114 // Produce histograms. Must be called after DecoderThreadStopped(), typically 115 // at the end of the call. 116 void UpdateHistograms(absl::optional<int> fraction_lost, 117 const StreamDataCounters& rtp_stats, 118 const StreamDataCounters* rtx_stats); 119 120 private: 121 struct QpCounters { 122 rtc::SampleCounter vp8; 123 }; 124 125 struct ContentSpecificStats { 126 ContentSpecificStats(); 127 ~ContentSpecificStats(); 128 129 void Add(const ContentSpecificStats& other); 130 131 rtc::SampleCounter e2e_delay_counter; 132 rtc::SampleCounter interframe_delay_counter; 133 int64_t flow_duration_ms = 0; 134 int64_t total_media_bytes = 0; 135 rtc::SampleCounter received_width; 136 rtc::SampleCounter received_height; 137 rtc::SampleCounter qp_counter; 138 FrameCounts frame_counts; 139 rtc::HistogramPercentileCounter interframe_delay_percentiles; 140 }; 141 142 void QualitySample(Timestamp now); 143 144 // Removes info about old frames and then updates the framerate. 145 void UpdateFramerate(int64_t now_ms) const; 146 147 void UpdateDecodeTimeHistograms(int width, 148 int height, 149 int decode_time_ms) const; 150 151 absl::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs( 152 int64_t now_ms) const; 153 154 Clock* const clock_; 155 const int64_t start_ms_; 156 const bool enable_decode_time_histograms_; 157 158 int64_t last_sample_time_ RTC_GUARDED_BY(main_thread_); 159 160 QualityThreshold fps_threshold_ RTC_GUARDED_BY(main_thread_); 161 QualityThreshold qp_threshold_ RTC_GUARDED_BY(main_thread_); 162 QualityThreshold variance_threshold_ RTC_GUARDED_BY(main_thread_); 163 rtc::SampleCounter qp_sample_ RTC_GUARDED_BY(main_thread_); 164 int num_bad_states_ RTC_GUARDED_BY(main_thread_); 165 int num_certain_states_ RTC_GUARDED_BY(main_thread_); 166 // Note: The |stats_.rtp_stats| member is not used or populated by this class. 167 mutable VideoReceiveStream::Stats stats_ RTC_GUARDED_BY(main_thread_); 168 // Same as stats_.ssrc, but const (no lock required). 169 const uint32_t remote_ssrc_; 170 RateStatistics decode_fps_estimator_ RTC_GUARDED_BY(main_thread_); 171 RateStatistics renders_fps_estimator_ RTC_GUARDED_BY(main_thread_); 172 rtc::RateTracker render_fps_tracker_ RTC_GUARDED_BY(main_thread_); 173 rtc::RateTracker render_pixel_tracker_ RTC_GUARDED_BY(main_thread_); 174 rtc::SampleCounter sync_offset_counter_ RTC_GUARDED_BY(main_thread_); 175 rtc::SampleCounter decode_time_counter_ RTC_GUARDED_BY(main_thread_); 176 rtc::SampleCounter jitter_buffer_delay_counter_ RTC_GUARDED_BY(main_thread_); 177 rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(main_thread_); 178 rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(main_thread_); 179 rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(main_thread_); 180 std::unique_ptr<VideoQualityObserver> video_quality_observer_ 181 RTC_GUARDED_BY(main_thread_); 182 mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_ 183 RTC_GUARDED_BY(main_thread_); 184 std::map<VideoContentType, ContentSpecificStats> content_specific_stats_ 185 RTC_GUARDED_BY(main_thread_); 186 MaxCounter freq_offset_counter_ RTC_GUARDED_BY(main_thread_); 187 QpCounters qp_counters_ RTC_GUARDED_BY(main_thread_); 188 int64_t avg_rtt_ms_ RTC_GUARDED_BY(main_thread_) = 0; 189 mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(main_thread_); 190 VideoContentType last_content_type_ RTC_GUARDED_BY(&main_thread_); 191 VideoCodecType last_codec_type_ RTC_GUARDED_BY(main_thread_); 192 absl::optional<int64_t> first_frame_received_time_ms_ 193 RTC_GUARDED_BY(main_thread_); 194 absl::optional<int64_t> first_decoded_frame_time_ms_ 195 RTC_GUARDED_BY(main_thread_); 196 absl::optional<int64_t> last_decoded_frame_time_ms_ 197 RTC_GUARDED_BY(main_thread_); 198 size_t num_delayed_frames_rendered_ RTC_GUARDED_BY(main_thread_); 199 int64_t sum_missed_render_deadline_ms_ RTC_GUARDED_BY(main_thread_); 200 // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is 201 // called from const GetStats(). 202 mutable rtc::MovingMaxCounter<TimingFrameInfo> timing_frame_info_counter_ 203 RTC_GUARDED_BY(main_thread_); 204 absl::optional<int> num_unique_frames_ RTC_GUARDED_BY(main_thread_); 205 absl::optional<int64_t> last_estimated_playout_ntp_timestamp_ms_ 206 RTC_GUARDED_BY(main_thread_); 207 absl::optional<int64_t> last_estimated_playout_time_ms_ 208 RTC_GUARDED_BY(main_thread_); 209 210 // The thread on which this instance is constructed and some of its main 211 // methods are invoked on such as GetStats(). 212 TaskQueueBase* const worker_thread_; 213 214 ScopedTaskSafety task_safety_; 215 216 SequenceChecker decode_queue_; 217 rtc::ThreadChecker main_thread_; 218 SequenceChecker incoming_render_queue_; 219 }; 220 221 } // namespace internal 222 } // namespace webrtc 223 #endif // VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 224