1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "api/video_codecs/video_decoder_software_fallback_wrapper.h"
12
13 #include <stdint.h>
14
15 #include <memory>
16 #include <string>
17 #include <utility>
18
19 #include "absl/base/macros.h"
20 #include "api/video/encoded_image.h"
21 #include "api/video_codecs/video_codec.h"
22 #include "modules/video_coding/include/video_error_codes.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/trace_event.h"
26 #include "system_wrappers/include/field_trial.h"
27 #include "system_wrappers/include/metrics.h"
28
29 namespace webrtc {
30
31 namespace {
32
33 constexpr size_t kMaxConsequtiveHwErrors = 4;
34
35 class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder {
36 public:
37 VideoDecoderSoftwareFallbackWrapper(
38 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
39 std::unique_ptr<VideoDecoder> hw_decoder);
40 ~VideoDecoderSoftwareFallbackWrapper() override;
41
42 int32_t InitDecode(const VideoCodec* codec_settings,
43 int32_t number_of_cores) override;
44
45 int32_t Decode(const EncodedImage& input_image,
46 bool missing_frames,
47 int64_t render_time_ms) override;
48
49 int32_t RegisterDecodeCompleteCallback(
50 DecodedImageCallback* callback) override;
51
52 int32_t Release() override;
53 bool PrefersLateDecoding() const override;
54
55 const char* ImplementationName() const override;
56
57 private:
58 bool InitFallbackDecoder();
59 void UpdateFallbackDecoderHistograms();
60
61 int32_t InitHwDecoder();
62
63 VideoDecoder& active_decoder() const;
64
65 // Determines if we are trying to use the HW or SW decoder.
66 enum class DecoderType {
67 kNone,
68 kHardware,
69 kFallback,
70 } decoder_type_;
71 std::unique_ptr<VideoDecoder> hw_decoder_;
72
73 VideoCodec codec_settings_;
74 int32_t number_of_cores_;
75 const std::unique_ptr<VideoDecoder> fallback_decoder_;
76 const std::string fallback_implementation_name_;
77 DecodedImageCallback* callback_;
78 int32_t hw_decoded_frames_since_last_fallback_;
79 size_t hw_consequtive_generic_errors_;
80 };
81
VideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)82 VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
83 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
84 std::unique_ptr<VideoDecoder> hw_decoder)
85 : decoder_type_(DecoderType::kNone),
86 hw_decoder_(std::move(hw_decoder)),
87 fallback_decoder_(std::move(sw_fallback_decoder)),
88 fallback_implementation_name_(
89 std::string(fallback_decoder_->ImplementationName()) +
90 " (fallback from: " + hw_decoder_->ImplementationName() + ")"),
91 callback_(nullptr),
92 hw_decoded_frames_since_last_fallback_(0),
93 hw_consequtive_generic_errors_(0) {}
94 VideoDecoderSoftwareFallbackWrapper::~VideoDecoderSoftwareFallbackWrapper() =
95 default;
96
InitDecode(const VideoCodec * codec_settings,int32_t number_of_cores)97 int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
98 const VideoCodec* codec_settings,
99 int32_t number_of_cores) {
100 codec_settings_ = *codec_settings;
101 number_of_cores_ = number_of_cores;
102
103 if (webrtc::field_trial::IsEnabled("WebRTC-Video-ForcedSwDecoderFallback")) {
104 RTC_LOG(LS_INFO) << "Forced software decoder fallback enabled.";
105 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
106 return InitFallbackDecoder() ? WEBRTC_VIDEO_CODEC_OK
107 : WEBRTC_VIDEO_CODEC_ERROR;
108 }
109 int32_t status = InitHwDecoder();
110 if (status == WEBRTC_VIDEO_CODEC_OK) {
111 return WEBRTC_VIDEO_CODEC_OK;
112 }
113
114 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
115 if (InitFallbackDecoder()) {
116 return WEBRTC_VIDEO_CODEC_OK;
117 }
118
119 return status;
120 }
121
InitHwDecoder()122 int32_t VideoDecoderSoftwareFallbackWrapper::InitHwDecoder() {
123 RTC_DCHECK(decoder_type_ == DecoderType::kNone);
124 int32_t status = hw_decoder_->InitDecode(&codec_settings_, number_of_cores_);
125 if (status != WEBRTC_VIDEO_CODEC_OK) {
126 return status;
127 }
128
129 decoder_type_ = DecoderType::kHardware;
130 if (callback_)
131 hw_decoder_->RegisterDecodeCompleteCallback(callback_);
132 return status;
133 }
134
InitFallbackDecoder()135 bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
136 RTC_DCHECK(decoder_type_ == DecoderType::kNone ||
137 decoder_type_ == DecoderType::kHardware);
138 RTC_LOG(LS_WARNING) << "Decoder falling back to software decoding.";
139 int32_t status =
140 fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_);
141 if (status != WEBRTC_VIDEO_CODEC_OK) {
142 RTC_LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
143 return false;
144 }
145
146 UpdateFallbackDecoderHistograms();
147
148 if (decoder_type_ == DecoderType::kHardware) {
149 hw_decoder_->Release();
150 }
151 decoder_type_ = DecoderType::kFallback;
152
153 if (callback_)
154 fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
155 return true;
156 }
157
UpdateFallbackDecoderHistograms()158 void VideoDecoderSoftwareFallbackWrapper::UpdateFallbackDecoderHistograms() {
159 const std::string kFallbackHistogramsUmaPrefix =
160 "WebRTC.Video.HardwareDecodedFramesBetweenSoftwareFallbacks.";
161 // Each histogram needs its own code path for this to work otherwise the
162 // histogram names will be mixed up by the optimization that takes place.
163 switch (codec_settings_.codecType) {
164 case kVideoCodecGeneric:
165 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Generic",
166 hw_decoded_frames_since_last_fallback_);
167 break;
168 case kVideoCodecVP8:
169 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp8",
170 hw_decoded_frames_since_last_fallback_);
171 break;
172 case kVideoCodecVP9:
173 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp9",
174 hw_decoded_frames_since_last_fallback_);
175 break;
176 case kVideoCodecAV1:
177 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Av1",
178 hw_decoded_frames_since_last_fallback_);
179 break;
180 case kVideoCodecH264:
181 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H264",
182 hw_decoded_frames_since_last_fallback_);
183 break;
184 case kVideoCodecMultiplex:
185 RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Multiplex",
186 hw_decoded_frames_since_last_fallback_);
187 break;
188 }
189 }
190
Decode(const EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)191 int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
192 const EncodedImage& input_image,
193 bool missing_frames,
194 int64_t render_time_ms) {
195 TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
196 switch (decoder_type_) {
197 case DecoderType::kNone:
198 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
199 case DecoderType::kHardware: {
200 int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
201 ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms);
202 if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
203 if (ret != WEBRTC_VIDEO_CODEC_ERROR) {
204 ++hw_decoded_frames_since_last_fallback_;
205 hw_consequtive_generic_errors_ = 0;
206 return ret;
207 }
208 if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
209 // Only count errors on key-frames, since generic errors can happen
210 // with hw decoder due to many arbitrary reasons.
211 // However, requesting a key-frame is supposed to fix the issue.
212 ++hw_consequtive_generic_errors_;
213 }
214 if (hw_consequtive_generic_errors_ < kMaxConsequtiveHwErrors) {
215 return ret;
216 }
217 }
218
219 // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or
220 // too many generic errors on key-frames encountered.
221 if (!InitFallbackDecoder()) {
222 return ret;
223 }
224
225 // Fallback decoder initialized, fall-through.
226 ABSL_FALLTHROUGH_INTENDED;
227 }
228 case DecoderType::kFallback:
229 return fallback_decoder_->Decode(input_image, missing_frames,
230 render_time_ms);
231 default:
232 RTC_NOTREACHED();
233 return WEBRTC_VIDEO_CODEC_ERROR;
234 }
235 }
236
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)237 int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
238 DecodedImageCallback* callback) {
239 callback_ = callback;
240 return active_decoder().RegisterDecodeCompleteCallback(callback);
241 }
242
Release()243 int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
244 int32_t status;
245 switch (decoder_type_) {
246 case DecoderType::kHardware:
247 status = hw_decoder_->Release();
248 break;
249 case DecoderType::kFallback:
250 RTC_LOG(LS_INFO) << "Releasing software fallback decoder.";
251 status = fallback_decoder_->Release();
252 break;
253 case DecoderType::kNone:
254 status = WEBRTC_VIDEO_CODEC_OK;
255 break;
256 default:
257 RTC_NOTREACHED();
258 status = WEBRTC_VIDEO_CODEC_ERROR;
259 }
260
261 decoder_type_ = DecoderType::kNone;
262 return status;
263 }
264
PrefersLateDecoding() const265 bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const {
266 return active_decoder().PrefersLateDecoding();
267 }
268
ImplementationName() const269 const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const {
270 return decoder_type_ == DecoderType::kFallback
271 ? fallback_implementation_name_.c_str()
272 : hw_decoder_->ImplementationName();
273 }
274
active_decoder() const275 VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const {
276 return decoder_type_ == DecoderType::kFallback ? *fallback_decoder_
277 : *hw_decoder_;
278 }
279
280 } // namespace
281
CreateVideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)282 std::unique_ptr<VideoDecoder> CreateVideoDecoderSoftwareFallbackWrapper(
283 std::unique_ptr<VideoDecoder> sw_fallback_decoder,
284 std::unique_ptr<VideoDecoder> hw_decoder) {
285 return std::make_unique<VideoDecoderSoftwareFallbackWrapper>(
286 std::move(sw_fallback_decoder), std::move(hw_decoder));
287 }
288
289 } // namespace webrtc
290