1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_streams_synchronizer.h"
12
13 #include "absl/types/optional.h"
14 #include "call/syncable.h"
15 #include "rtc_base/checks.h"
16 #include "rtc_base/logging.h"
17 #include "rtc_base/time_utils.h"
18 #include "rtc_base/trace_event.h"
19 #include "system_wrappers/include/rtp_to_ntp_estimator.h"
20
21 namespace webrtc {
22 namespace {
23 // Time interval for logging stats.
24 constexpr int64_t kStatsLogIntervalMs = 10000;
25
UpdateMeasurements(StreamSynchronization::Measurements * stream,const Syncable::Info & info)26 bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
27 const Syncable::Info& info) {
28 RTC_DCHECK(stream);
29 stream->latest_timestamp = info.latest_received_capture_timestamp;
30 stream->latest_receive_time_ms = info.latest_receive_time_ms;
31 bool new_rtcp_sr = false;
32 if (!stream->rtp_to_ntp.UpdateMeasurements(
33 info.capture_time_ntp_secs, info.capture_time_ntp_frac,
34 info.capture_time_source_clock, &new_rtcp_sr)) {
35 return false;
36 }
37 return true;
38 }
39 } // namespace
40
RtpStreamsSynchronizer(Syncable * syncable_video)41 RtpStreamsSynchronizer::RtpStreamsSynchronizer(Syncable* syncable_video)
42 : syncable_video_(syncable_video),
43 syncable_audio_(nullptr),
44 sync_(),
45 last_sync_time_(rtc::TimeNanos()),
46 last_stats_log_ms_(rtc::TimeMillis()) {
47 RTC_DCHECK(syncable_video);
48 process_thread_checker_.Detach();
49 }
50
51 RtpStreamsSynchronizer::~RtpStreamsSynchronizer() = default;
52
ConfigureSync(Syncable * syncable_audio)53 void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) {
54 MutexLock lock(&mutex_);
55 if (syncable_audio == syncable_audio_) {
56 // This prevents expensive no-ops.
57 return;
58 }
59
60 syncable_audio_ = syncable_audio;
61 sync_.reset(nullptr);
62 if (syncable_audio_) {
63 sync_.reset(new StreamSynchronization(syncable_video_->id(),
64 syncable_audio_->id()));
65 }
66 }
67
TimeUntilNextProcess()68 int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() {
69 RTC_DCHECK_RUN_ON(&process_thread_checker_);
70 const int64_t kSyncIntervalMs = 1000;
71 return kSyncIntervalMs -
72 (rtc::TimeNanos() - last_sync_time_) / rtc::kNumNanosecsPerMillisec;
73 }
74
Process()75 void RtpStreamsSynchronizer::Process() {
76 RTC_DCHECK_RUN_ON(&process_thread_checker_);
77 last_sync_time_ = rtc::TimeNanos();
78
79 MutexLock lock(&mutex_);
80 if (!syncable_audio_) {
81 return;
82 }
83 RTC_DCHECK(sync_.get());
84
85 bool log_stats = false;
86 const int64_t now_ms = rtc::TimeMillis();
87 if (now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
88 last_stats_log_ms_ = now_ms;
89 log_stats = true;
90 }
91
92 absl::optional<Syncable::Info> audio_info = syncable_audio_->GetInfo();
93 if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) {
94 return;
95 }
96
97 int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
98 absl::optional<Syncable::Info> video_info = syncable_video_->GetInfo();
99 if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) {
100 return;
101 }
102
103 if (last_video_receive_ms == video_measurement_.latest_receive_time_ms) {
104 // No new video packet has been received since last update.
105 return;
106 }
107
108 int relative_delay_ms;
109 // Calculate how much later or earlier the audio stream is compared to video.
110 if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_,
111 &relative_delay_ms)) {
112 return;
113 }
114
115 if (log_stats) {
116 RTC_LOG(LS_INFO) << "Sync info stats: " << now_ms
117 << ", {ssrc: " << sync_->audio_stream_id() << ", "
118 << "cur_delay_ms: " << audio_info->current_delay_ms
119 << "} {ssrc: " << sync_->video_stream_id() << ", "
120 << "cur_delay_ms: " << video_info->current_delay_ms
121 << "} {relative_delay_ms: " << relative_delay_ms << "} ";
122 }
123
124 TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay",
125 video_info->current_delay_ms);
126 TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay",
127 audio_info->current_delay_ms);
128 TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
129
130 int target_audio_delay_ms = 0;
131 int target_video_delay_ms = video_info->current_delay_ms;
132 // Calculate the necessary extra audio delay and desired total video
133 // delay to get the streams in sync.
134 if (!sync_->ComputeDelays(relative_delay_ms, audio_info->current_delay_ms,
135 &target_audio_delay_ms, &target_video_delay_ms)) {
136 return;
137 }
138
139 if (log_stats) {
140 RTC_LOG(LS_INFO) << "Sync delay stats: " << now_ms
141 << ", {ssrc: " << sync_->audio_stream_id() << ", "
142 << "target_delay_ms: " << target_audio_delay_ms
143 << "} {ssrc: " << sync_->video_stream_id() << ", "
144 << "target_delay_ms: " << target_video_delay_ms << "} ";
145 }
146
147 syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms);
148 syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms);
149 }
150
151 // TODO(https://bugs.webrtc.org/7065): Move RtpToNtpEstimator out of
152 // RtpStreamsSynchronizer and into respective receive stream to always populate
153 // the estimated playout timestamp.
GetStreamSyncOffsetInMs(uint32_t rtp_timestamp,int64_t render_time_ms,int64_t * video_playout_ntp_ms,int64_t * stream_offset_ms,double * estimated_freq_khz) const154 bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
155 uint32_t rtp_timestamp,
156 int64_t render_time_ms,
157 int64_t* video_playout_ntp_ms,
158 int64_t* stream_offset_ms,
159 double* estimated_freq_khz) const {
160 MutexLock lock(&mutex_);
161 if (!syncable_audio_) {
162 return false;
163 }
164
165 uint32_t audio_rtp_timestamp;
166 int64_t time_ms;
167 if (!syncable_audio_->GetPlayoutRtpTimestamp(&audio_rtp_timestamp,
168 &time_ms)) {
169 return false;
170 }
171
172 int64_t latest_audio_ntp;
173 if (!audio_measurement_.rtp_to_ntp.Estimate(audio_rtp_timestamp,
174 &latest_audio_ntp)) {
175 return false;
176 }
177
178 syncable_audio_->SetEstimatedPlayoutNtpTimestampMs(latest_audio_ntp, time_ms);
179
180 int64_t latest_video_ntp;
181 if (!video_measurement_.rtp_to_ntp.Estimate(rtp_timestamp,
182 &latest_video_ntp)) {
183 return false;
184 }
185
186 // Current audio ntp.
187 int64_t now_ms = rtc::TimeMillis();
188 latest_audio_ntp += (now_ms - time_ms);
189
190 // Remove video playout delay.
191 int64_t time_to_render_ms = render_time_ms - now_ms;
192 if (time_to_render_ms > 0)
193 latest_video_ntp -= time_to_render_ms;
194
195 *video_playout_ntp_ms = latest_video_ntp;
196 *stream_offset_ms = latest_audio_ntp - latest_video_ntp;
197 *estimated_freq_khz = video_measurement_.rtp_to_ntp.params()->frequency_khz;
198 return true;
199 }
200
201 } // namespace webrtc
202