1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_RECEIVE_STATISTICS_PROXY2_H_
12 #define VIDEO_RECEIVE_STATISTICS_PROXY2_H_
13 
14 #include <map>
15 #include <memory>
16 #include <string>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/task_queue/task_queue_base.h"
21 #include "api/units/timestamp.h"
22 #include "call/video_receive_stream.h"
23 #include "modules/include/module_common_types.h"
24 #include "modules/video_coding/include/video_coding_defines.h"
25 #include "rtc_base/numerics/histogram_percentile_counter.h"
26 #include "rtc_base/numerics/moving_max_counter.h"
27 #include "rtc_base/numerics/sample_counter.h"
28 #include "rtc_base/rate_statistics.h"
29 #include "rtc_base/rate_tracker.h"
30 #include "rtc_base/synchronization/sequence_checker.h"
31 #include "rtc_base/task_utils/pending_task_safety_flag.h"
32 #include "rtc_base/thread_annotations.h"
33 #include "rtc_base/thread_checker.h"
34 #include "video/quality_threshold.h"
35 #include "video/stats_counter.h"
36 #include "video/video_quality_observer2.h"
37 
38 namespace webrtc {
39 
40 class Clock;
41 struct CodecSpecificInfo;
42 
43 namespace internal {
44 // Declared in video_receive_stream2.h.
45 struct VideoFrameMetaData;
46 
47 class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
48                                public RtcpCnameCallback,
49                                public RtcpPacketTypeCounterObserver {
50  public:
51   ReceiveStatisticsProxy(const VideoReceiveStream::Config* config,
52                          Clock* clock,
53                          TaskQueueBase* worker_thread);
54   ~ReceiveStatisticsProxy() override;
55 
56   VideoReceiveStream::Stats GetStats() const;
57 
58   void OnDecodedFrame(const VideoFrame& frame,
59                       absl::optional<uint8_t> qp,
60                       int32_t decode_time_ms,
61                       VideoContentType content_type);
62 
63   // Called asyncronously on the worker thread as a result of a call to the
64   // above OnDecodedFrame method, which is called back on the thread where
65   // the actual decoding happens.
66   void OnDecodedFrame(const VideoFrameMetaData& frame_meta,
67                       absl::optional<uint8_t> qp,
68                       int32_t decode_time_ms,
69                       VideoContentType content_type);
70 
71   void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms,
72                            int64_t sync_offset_ms,
73                            double estimated_freq_khz);
74   void OnRenderedFrame(const VideoFrameMetaData& frame_meta);
75   void OnIncomingPayloadType(int payload_type);
76   void OnDecoderImplementationName(const char* implementation_name);
77 
78   void OnPreDecode(VideoCodecType codec_type, int qp);
79 
80   void OnUniqueFramesCounted(int num_unique_frames);
81 
82   // Indicates video stream has been paused (no incoming packets).
83   void OnStreamInactive();
84 
85   // Overrides VCMReceiveStatisticsCallback.
86   void OnCompleteFrame(bool is_keyframe,
87                        size_t size_bytes,
88                        VideoContentType content_type) override;
89   void OnDroppedFrames(uint32_t frames_dropped) override;
90   void OnFrameBufferTimingsUpdated(int max_decode_ms,
91                                    int current_delay_ms,
92                                    int target_delay_ms,
93                                    int jitter_buffer_ms,
94                                    int min_playout_delay_ms,
95                                    int render_delay_ms) override;
96 
97   void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override;
98 
99   // Overrides RtcpCnameCallback.
100   void OnCname(uint32_t ssrc, absl::string_view cname) override;
101 
102   // Overrides RtcpPacketTypeCounterObserver.
103   void RtcpPacketTypesCounterUpdated(
104       uint32_t ssrc,
105       const RtcpPacketTypeCounter& packet_counter) override;
106 
107   void OnRttUpdate(int64_t avg_rtt_ms);
108 
109   // Notification methods that are used to check our internal state and validate
110   // threading assumptions. These are called by VideoReceiveStream.
111   void DecoderThreadStarting();
112   void DecoderThreadStopped();
113 
114   // Produce histograms. Must be called after DecoderThreadStopped(), typically
115   // at the end of the call.
116   void UpdateHistograms(absl::optional<int> fraction_lost,
117                         const StreamDataCounters& rtp_stats,
118                         const StreamDataCounters* rtx_stats);
119 
120  private:
121   struct QpCounters {
122     rtc::SampleCounter vp8;
123   };
124 
125   struct ContentSpecificStats {
126     ContentSpecificStats();
127     ~ContentSpecificStats();
128 
129     void Add(const ContentSpecificStats& other);
130 
131     rtc::SampleCounter e2e_delay_counter;
132     rtc::SampleCounter interframe_delay_counter;
133     int64_t flow_duration_ms = 0;
134     int64_t total_media_bytes = 0;
135     rtc::SampleCounter received_width;
136     rtc::SampleCounter received_height;
137     rtc::SampleCounter qp_counter;
138     FrameCounts frame_counts;
139     rtc::HistogramPercentileCounter interframe_delay_percentiles;
140   };
141 
142   void QualitySample(Timestamp now);
143 
144   // Removes info about old frames and then updates the framerate.
145   void UpdateFramerate(int64_t now_ms) const;
146 
147   void UpdateDecodeTimeHistograms(int width,
148                                   int height,
149                                   int decode_time_ms) const;
150 
151   absl::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs(
152       int64_t now_ms) const;
153 
154   Clock* const clock_;
155   const int64_t start_ms_;
156   const bool enable_decode_time_histograms_;
157 
158   int64_t last_sample_time_ RTC_GUARDED_BY(main_thread_);
159 
160   QualityThreshold fps_threshold_ RTC_GUARDED_BY(main_thread_);
161   QualityThreshold qp_threshold_ RTC_GUARDED_BY(main_thread_);
162   QualityThreshold variance_threshold_ RTC_GUARDED_BY(main_thread_);
163   rtc::SampleCounter qp_sample_ RTC_GUARDED_BY(main_thread_);
164   int num_bad_states_ RTC_GUARDED_BY(main_thread_);
165   int num_certain_states_ RTC_GUARDED_BY(main_thread_);
166   // Note: The |stats_.rtp_stats| member is not used or populated by this class.
167   mutable VideoReceiveStream::Stats stats_ RTC_GUARDED_BY(main_thread_);
168   // Same as stats_.ssrc, but const (no lock required).
169   const uint32_t remote_ssrc_;
170   RateStatistics decode_fps_estimator_ RTC_GUARDED_BY(main_thread_);
171   RateStatistics renders_fps_estimator_ RTC_GUARDED_BY(main_thread_);
172   rtc::RateTracker render_fps_tracker_ RTC_GUARDED_BY(main_thread_);
173   rtc::RateTracker render_pixel_tracker_ RTC_GUARDED_BY(main_thread_);
174   rtc::SampleCounter sync_offset_counter_ RTC_GUARDED_BY(main_thread_);
175   rtc::SampleCounter decode_time_counter_ RTC_GUARDED_BY(main_thread_);
176   rtc::SampleCounter jitter_buffer_delay_counter_ RTC_GUARDED_BY(main_thread_);
177   rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(main_thread_);
178   rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(main_thread_);
179   rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(main_thread_);
180   std::unique_ptr<VideoQualityObserver> video_quality_observer_
181       RTC_GUARDED_BY(main_thread_);
182   mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_
183       RTC_GUARDED_BY(main_thread_);
184   std::map<VideoContentType, ContentSpecificStats> content_specific_stats_
185       RTC_GUARDED_BY(main_thread_);
186   MaxCounter freq_offset_counter_ RTC_GUARDED_BY(main_thread_);
187   QpCounters qp_counters_ RTC_GUARDED_BY(main_thread_);
188   int64_t avg_rtt_ms_ RTC_GUARDED_BY(main_thread_) = 0;
189   mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(main_thread_);
190   VideoContentType last_content_type_ RTC_GUARDED_BY(&main_thread_);
191   VideoCodecType last_codec_type_ RTC_GUARDED_BY(main_thread_);
192   absl::optional<int64_t> first_frame_received_time_ms_
193       RTC_GUARDED_BY(main_thread_);
194   absl::optional<int64_t> first_decoded_frame_time_ms_
195       RTC_GUARDED_BY(main_thread_);
196   absl::optional<int64_t> last_decoded_frame_time_ms_
197       RTC_GUARDED_BY(main_thread_);
198   size_t num_delayed_frames_rendered_ RTC_GUARDED_BY(main_thread_);
199   int64_t sum_missed_render_deadline_ms_ RTC_GUARDED_BY(main_thread_);
200   // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is
201   // called from const GetStats().
202   mutable rtc::MovingMaxCounter<TimingFrameInfo> timing_frame_info_counter_
203       RTC_GUARDED_BY(main_thread_);
204   absl::optional<int> num_unique_frames_ RTC_GUARDED_BY(main_thread_);
205   absl::optional<int64_t> last_estimated_playout_ntp_timestamp_ms_
206       RTC_GUARDED_BY(main_thread_);
207   absl::optional<int64_t> last_estimated_playout_time_ms_
208       RTC_GUARDED_BY(main_thread_);
209 
210   // The thread on which this instance is constructed and some of its main
211   // methods are invoked on such as GetStats().
212   TaskQueueBase* const worker_thread_;
213 
214   ScopedTaskSafety task_safety_;
215 
216   SequenceChecker decode_queue_;
217   rtc::ThreadChecker main_thread_;
218   SequenceChecker incoming_render_queue_;
219 };
220 
221 }  // namespace internal
222 }  // namespace webrtc
223 #endif  // VIDEO_RECEIVE_STATISTICS_PROXY2_H_
224