1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/generic_decoder.h"
12
13 #include <stddef.h>
14
15 #include <algorithm>
16
17 #include "api/video/video_timing.h"
18 #include "modules/video_coding/include/video_error_codes.h"
19 #include "rtc_base/checks.h"
20 #include "rtc_base/logging.h"
21 #include "rtc_base/thread.h"
22 #include "rtc_base/time_utils.h"
23 #include "rtc_base/trace_event.h"
24 #include "system_wrappers/include/clock.h"
25 #include "system_wrappers/include/field_trial.h"
26
27 namespace webrtc {
28
VCMDecodedFrameCallback(VCMTiming * timing,Clock * clock)29 VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing,
30 Clock* clock)
31 : _clock(clock),
32 _timing(timing),
33 _timestampMap(kDecoderFrameMemoryLength),
34 _extra_decode_time("t", absl::nullopt) {
35 ntp_offset_ =
36 _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
37
38 ParseFieldTrial({&_extra_decode_time},
39 field_trial::FindFullName("WebRTC-SlowDownDecoder"));
40 }
41
~VCMDecodedFrameCallback()42 VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}
43
SetUserReceiveCallback(VCMReceiveCallback * receiveCallback)44 void VCMDecodedFrameCallback::SetUserReceiveCallback(
45 VCMReceiveCallback* receiveCallback) {
46 RTC_DCHECK(construction_thread_.IsCurrent());
47 RTC_DCHECK((!_receiveCallback && receiveCallback) ||
48 (_receiveCallback && !receiveCallback));
49 _receiveCallback = receiveCallback;
50 }
51
UserReceiveCallback()52 VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
53 // Called on the decode thread via VCMCodecDataBase::GetDecoder.
54 // The callback must always have been set before this happens.
55 RTC_DCHECK(_receiveCallback);
56 return _receiveCallback;
57 }
58
Decoded(VideoFrame & decodedImage)59 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
60 // This function may be called on the decode TaskQueue, but may also be called
61 // on an OS provided queue such as on iOS (see e.g. b/153465112).
62 return Decoded(decodedImage, -1);
63 }
64
Decoded(VideoFrame & decodedImage,int64_t decode_time_ms)65 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
66 int64_t decode_time_ms) {
67 Decoded(decodedImage,
68 decode_time_ms >= 0 ? absl::optional<int32_t>(decode_time_ms)
69 : absl::nullopt,
70 absl::nullopt);
71 return WEBRTC_VIDEO_CODEC_OK;
72 }
73
Decoded(VideoFrame & decodedImage,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)74 void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
75 absl::optional<int32_t> decode_time_ms,
76 absl::optional<uint8_t> qp) {
77 // Wait some extra time to simulate a slow decoder.
78 if (_extra_decode_time) {
79 rtc::Thread::SleepMs(_extra_decode_time->ms());
80 }
81
82 RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
83 TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
84 "timestamp", decodedImage.timestamp());
85 // TODO(holmer): We should improve this so that we can handle multiple
86 // callbacks from one call to Decode().
87 VCMFrameInformation* frameInfo;
88 {
89 MutexLock lock(&lock_);
90 frameInfo = _timestampMap.Pop(decodedImage.timestamp());
91 }
92
93 if (frameInfo == NULL) {
94 RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
95 "this one.";
96 _receiveCallback->OnDroppedFrames(1);
97 return;
98 }
99
100 decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
101 decodedImage.set_packet_infos(frameInfo->packet_infos);
102 decodedImage.set_rotation(frameInfo->rotation);
103
104 const Timestamp now = _clock->CurrentTime();
105 RTC_DCHECK(frameInfo->decodeStart);
106 if (!decode_time_ms) {
107 decode_time_ms = (now - *frameInfo->decodeStart).ms();
108 }
109 _timing->StopDecodeTimer(*decode_time_ms, now.ms());
110 decodedImage.set_processing_time({*frameInfo->decodeStart, now});
111
112 // Report timing information.
113 TimingFrameInfo timing_frame_info;
114 if (frameInfo->timing.flags != VideoSendTiming::kInvalid) {
115 int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
116 // Convert remote timestamps to local time from ntp timestamps.
117 frameInfo->timing.encode_start_ms -= ntp_offset_;
118 frameInfo->timing.encode_finish_ms -= ntp_offset_;
119 frameInfo->timing.packetization_finish_ms -= ntp_offset_;
120 frameInfo->timing.pacer_exit_ms -= ntp_offset_;
121 frameInfo->timing.network_timestamp_ms -= ntp_offset_;
122 frameInfo->timing.network2_timestamp_ms -= ntp_offset_;
123
124 int64_t sender_delta_ms = 0;
125 if (decodedImage.ntp_time_ms() < 0) {
126 // Sender clock is not estimated yet. Make sure that sender times are all
127 // negative to indicate that. Yet they still should be relatively correct.
128 sender_delta_ms =
129 std::max({capture_time_ms, frameInfo->timing.encode_start_ms,
130 frameInfo->timing.encode_finish_ms,
131 frameInfo->timing.packetization_finish_ms,
132 frameInfo->timing.pacer_exit_ms,
133 frameInfo->timing.network_timestamp_ms,
134 frameInfo->timing.network2_timestamp_ms}) +
135 1;
136 }
137
138 timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
139 timing_frame_info.encode_start_ms =
140 frameInfo->timing.encode_start_ms - sender_delta_ms;
141 timing_frame_info.encode_finish_ms =
142 frameInfo->timing.encode_finish_ms - sender_delta_ms;
143 timing_frame_info.packetization_finish_ms =
144 frameInfo->timing.packetization_finish_ms - sender_delta_ms;
145 timing_frame_info.pacer_exit_ms =
146 frameInfo->timing.pacer_exit_ms - sender_delta_ms;
147 timing_frame_info.network_timestamp_ms =
148 frameInfo->timing.network_timestamp_ms - sender_delta_ms;
149 timing_frame_info.network2_timestamp_ms =
150 frameInfo->timing.network2_timestamp_ms - sender_delta_ms;
151 }
152
153 timing_frame_info.flags = frameInfo->timing.flags;
154 timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
155 timing_frame_info.decode_finish_ms = now.ms();
156 timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
157 timing_frame_info.rtp_timestamp = decodedImage.timestamp();
158 timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
159 timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms;
160 _timing->SetTimingFrameInfo(timing_frame_info);
161
162 decodedImage.set_timestamp_us(frameInfo->renderTimeMs *
163 rtc::kNumMicrosecsPerMillisec);
164 _receiveCallback->FrameToRender(decodedImage, qp, *decode_time_ms,
165 frameInfo->content_type);
166 }
167
OnDecoderImplementationName(const char * implementation_name)168 void VCMDecodedFrameCallback::OnDecoderImplementationName(
169 const char* implementation_name) {
170 _receiveCallback->OnDecoderImplementationName(implementation_name);
171 }
172
Map(uint32_t timestamp,VCMFrameInformation * frameInfo)173 void VCMDecodedFrameCallback::Map(uint32_t timestamp,
174 VCMFrameInformation* frameInfo) {
175 MutexLock lock(&lock_);
176 _timestampMap.Add(timestamp, frameInfo);
177 }
178
Pop(uint32_t timestamp)179 int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) {
180 MutexLock lock(&lock_);
181 if (_timestampMap.Pop(timestamp) == NULL) {
182 return VCM_GENERAL_ERROR;
183 }
184 _receiveCallback->OnDroppedFrames(1);
185 return VCM_OK;
186 }
187
VCMGenericDecoder(std::unique_ptr<VideoDecoder> decoder)188 VCMGenericDecoder::VCMGenericDecoder(std::unique_ptr<VideoDecoder> decoder)
189 : VCMGenericDecoder(decoder.release(), false /* isExternal */) {}
190
VCMGenericDecoder(VideoDecoder * decoder,bool isExternal)191 VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal)
192 : _callback(NULL),
193 _frameInfos(),
194 _nextFrameInfoIdx(0),
195 decoder_(decoder),
196 _codecType(kVideoCodecGeneric),
197 _isExternal(isExternal),
198 _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {
199 RTC_DCHECK(decoder_);
200 }
201
~VCMGenericDecoder()202 VCMGenericDecoder::~VCMGenericDecoder() {
203 decoder_->Release();
204 if (_isExternal)
205 decoder_.release();
206 RTC_DCHECK(_isExternal || decoder_);
207 }
208
InitDecode(const VideoCodec * settings,int32_t numberOfCores)209 int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings,
210 int32_t numberOfCores) {
211 TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode");
212 _codecType = settings->codecType;
213
214 int err = decoder_->InitDecode(settings, numberOfCores);
215 implementation_name_ = decoder_->ImplementationName();
216 RTC_LOG(LS_INFO) << "Decoder implementation: " << implementation_name_;
217 return err;
218 }
219
Decode(const VCMEncodedFrame & frame,Timestamp now)220 int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
221 TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
222 frame.Timestamp());
223 _frameInfos[_nextFrameInfoIdx].decodeStart = now;
224 _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
225 _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
226 _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing();
227 _frameInfos[_nextFrameInfoIdx].ntp_time_ms =
228 frame.EncodedImage().ntp_time_ms_;
229 _frameInfos[_nextFrameInfoIdx].packet_infos = frame.PacketInfos();
230
231 // Set correctly only for key frames. Thus, use latest key frame
232 // content type. If the corresponding key frame was lost, decode will fail
233 // and content type will be ignored.
234 if (frame.FrameType() == VideoFrameType::kVideoFrameKey) {
235 _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType();
236 _last_keyframe_content_type = frame.contentType();
237 } else {
238 _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type;
239 }
240 _callback->Map(frame.Timestamp(), &_frameInfos[_nextFrameInfoIdx]);
241
242 _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;
243 int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
244 frame.RenderTimeMs());
245 const char* new_implementation_name = decoder_->ImplementationName();
246 if (new_implementation_name != implementation_name_) {
247 implementation_name_ = new_implementation_name;
248 RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
249 << new_implementation_name;
250 }
251 _callback->OnDecoderImplementationName(implementation_name_.c_str());
252 if (ret < WEBRTC_VIDEO_CODEC_OK) {
253 RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
254 << frame.Timestamp() << ", error code: " << ret;
255 _callback->Pop(frame.Timestamp());
256 return ret;
257 } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) {
258 // No output
259 _callback->Pop(frame.Timestamp());
260 }
261 return ret;
262 }
263
RegisterDecodeCompleteCallback(VCMDecodedFrameCallback * callback)264 int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
265 VCMDecodedFrameCallback* callback) {
266 _callback = callback;
267 return decoder_->RegisterDecodeCompleteCallback(callback);
268 }
269
PrefersLateDecoding() const270 bool VCMGenericDecoder::PrefersLateDecoding() const {
271 return decoder_->PrefersLateDecoding();
272 }
273
274 } // namespace webrtc
275