1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "api/video_codecs/video_decoder_software_fallback_wrapper.h"
12 
13 #include <stdint.h>
14 
15 #include <memory>
16 #include <string>
17 #include <utility>
18 
19 #include "absl/base/macros.h"
20 #include "api/video/encoded_image.h"
21 #include "api/video_codecs/video_codec.h"
22 #include "modules/video_coding/include/video_error_codes.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/trace_event.h"
26 #include "system_wrappers/include/field_trial.h"
27 #include "system_wrappers/include/metrics.h"
28 
29 namespace webrtc {
30 
31 namespace {
32 
33 constexpr size_t kMaxConsequtiveHwErrors = 4;
34 
35 class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder {
36  public:
37   VideoDecoderSoftwareFallbackWrapper(
38       std::unique_ptr<VideoDecoder> sw_fallback_decoder,
39       std::unique_ptr<VideoDecoder> hw_decoder);
40   ~VideoDecoderSoftwareFallbackWrapper() override;
41 
42   int32_t InitDecode(const VideoCodec* codec_settings,
43                      int32_t number_of_cores) override;
44 
45   int32_t Decode(const EncodedImage& input_image,
46                  bool missing_frames,
47                  int64_t render_time_ms) override;
48 
49   int32_t RegisterDecodeCompleteCallback(
50       DecodedImageCallback* callback) override;
51 
52   int32_t Release() override;
53   bool PrefersLateDecoding() const override;
54 
55   const char* ImplementationName() const override;
56 
57  private:
58   bool InitFallbackDecoder();
59   void UpdateFallbackDecoderHistograms();
60 
61   int32_t InitHwDecoder();
62 
63   VideoDecoder& active_decoder() const;
64 
65   // Determines if we are trying to use the HW or SW decoder.
66   enum class DecoderType {
67     kNone,
68     kHardware,
69     kFallback,
70   } decoder_type_;
71   std::unique_ptr<VideoDecoder> hw_decoder_;
72 
73   VideoCodec codec_settings_;
74   int32_t number_of_cores_;
75   const std::unique_ptr<VideoDecoder> fallback_decoder_;
76   const std::string fallback_implementation_name_;
77   DecodedImageCallback* callback_;
78   int32_t hw_decoded_frames_since_last_fallback_;
79   size_t hw_consequtive_generic_errors_;
80 };
81 
VideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)82 VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
83     std::unique_ptr<VideoDecoder> sw_fallback_decoder,
84     std::unique_ptr<VideoDecoder> hw_decoder)
85     : decoder_type_(DecoderType::kNone),
86       hw_decoder_(std::move(hw_decoder)),
87       fallback_decoder_(std::move(sw_fallback_decoder)),
88       fallback_implementation_name_(
89           std::string(fallback_decoder_->ImplementationName()) +
90           " (fallback from: " + hw_decoder_->ImplementationName() + ")"),
91       callback_(nullptr),
92       hw_decoded_frames_since_last_fallback_(0),
93       hw_consequtive_generic_errors_(0) {}
94 VideoDecoderSoftwareFallbackWrapper::~VideoDecoderSoftwareFallbackWrapper() =
95     default;
96 
InitDecode(const VideoCodec * codec_settings,int32_t number_of_cores)97 int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
98     const VideoCodec* codec_settings,
99     int32_t number_of_cores) {
100   codec_settings_ = *codec_settings;
101   number_of_cores_ = number_of_cores;
102 
103   if (webrtc::field_trial::IsEnabled("WebRTC-Video-ForcedSwDecoderFallback")) {
104     RTC_LOG(LS_INFO) << "Forced software decoder fallback enabled.";
105     RTC_DCHECK(decoder_type_ == DecoderType::kNone);
106     return InitFallbackDecoder() ? WEBRTC_VIDEO_CODEC_OK
107                                  : WEBRTC_VIDEO_CODEC_ERROR;
108   }
109   int32_t status = InitHwDecoder();
110   if (status == WEBRTC_VIDEO_CODEC_OK) {
111     return WEBRTC_VIDEO_CODEC_OK;
112   }
113 
114   RTC_DCHECK(decoder_type_ == DecoderType::kNone);
115   if (InitFallbackDecoder()) {
116     return WEBRTC_VIDEO_CODEC_OK;
117   }
118 
119   return status;
120 }
121 
InitHwDecoder()122 int32_t VideoDecoderSoftwareFallbackWrapper::InitHwDecoder() {
123   RTC_DCHECK(decoder_type_ == DecoderType::kNone);
124   int32_t status = hw_decoder_->InitDecode(&codec_settings_, number_of_cores_);
125   if (status != WEBRTC_VIDEO_CODEC_OK) {
126     return status;
127   }
128 
129   decoder_type_ = DecoderType::kHardware;
130   if (callback_)
131     hw_decoder_->RegisterDecodeCompleteCallback(callback_);
132   return status;
133 }
134 
InitFallbackDecoder()135 bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
136   RTC_DCHECK(decoder_type_ == DecoderType::kNone ||
137              decoder_type_ == DecoderType::kHardware);
138   RTC_LOG(LS_WARNING) << "Decoder falling back to software decoding.";
139   int32_t status =
140       fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_);
141   if (status != WEBRTC_VIDEO_CODEC_OK) {
142     RTC_LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
143     return false;
144   }
145 
146   UpdateFallbackDecoderHistograms();
147 
148   if (decoder_type_ == DecoderType::kHardware) {
149     hw_decoder_->Release();
150   }
151   decoder_type_ = DecoderType::kFallback;
152 
153   if (callback_)
154     fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
155   return true;
156 }
157 
UpdateFallbackDecoderHistograms()158 void VideoDecoderSoftwareFallbackWrapper::UpdateFallbackDecoderHistograms() {
159   const std::string kFallbackHistogramsUmaPrefix =
160       "WebRTC.Video.HardwareDecodedFramesBetweenSoftwareFallbacks.";
161   // Each histogram needs its own code path for this to work otherwise the
162   // histogram names will be mixed up by the optimization that takes place.
163   switch (codec_settings_.codecType) {
164     case kVideoCodecGeneric:
165       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Generic",
166                                   hw_decoded_frames_since_last_fallback_);
167       break;
168     case kVideoCodecVP8:
169       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp8",
170                                   hw_decoded_frames_since_last_fallback_);
171       break;
172     case kVideoCodecVP9:
173       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp9",
174                                   hw_decoded_frames_since_last_fallback_);
175       break;
176     case kVideoCodecAV1:
177       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Av1",
178                                   hw_decoded_frames_since_last_fallback_);
179       break;
180     case kVideoCodecH264:
181       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H264",
182                                   hw_decoded_frames_since_last_fallback_);
183       break;
184     case kVideoCodecMultiplex:
185       RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Multiplex",
186                                   hw_decoded_frames_since_last_fallback_);
187       break;
188   }
189 }
190 
Decode(const EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)191 int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
192     const EncodedImage& input_image,
193     bool missing_frames,
194     int64_t render_time_ms) {
195   TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode");
196   switch (decoder_type_) {
197     case DecoderType::kNone:
198       return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
199     case DecoderType::kHardware: {
200       int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
201       ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms);
202       if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
203         if (ret != WEBRTC_VIDEO_CODEC_ERROR) {
204           ++hw_decoded_frames_since_last_fallback_;
205           hw_consequtive_generic_errors_ = 0;
206           return ret;
207         }
208         if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
209           // Only count errors on key-frames, since generic errors can happen
210           // with hw decoder due to many arbitrary reasons.
211           // However, requesting a key-frame is supposed to fix the issue.
212           ++hw_consequtive_generic_errors_;
213         }
214         if (hw_consequtive_generic_errors_ < kMaxConsequtiveHwErrors) {
215           return ret;
216         }
217       }
218 
219       // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or
220       // too many generic errors on key-frames encountered.
221       if (!InitFallbackDecoder()) {
222         return ret;
223       }
224 
225       // Fallback decoder initialized, fall-through.
226       ABSL_FALLTHROUGH_INTENDED;
227     }
228     case DecoderType::kFallback:
229       return fallback_decoder_->Decode(input_image, missing_frames,
230                                        render_time_ms);
231     default:
232       RTC_NOTREACHED();
233       return WEBRTC_VIDEO_CODEC_ERROR;
234   }
235 }
236 
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)237 int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
238     DecodedImageCallback* callback) {
239   callback_ = callback;
240   return active_decoder().RegisterDecodeCompleteCallback(callback);
241 }
242 
Release()243 int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
244   int32_t status;
245   switch (decoder_type_) {
246     case DecoderType::kHardware:
247       status = hw_decoder_->Release();
248       break;
249     case DecoderType::kFallback:
250       RTC_LOG(LS_INFO) << "Releasing software fallback decoder.";
251       status = fallback_decoder_->Release();
252       break;
253     case DecoderType::kNone:
254       status = WEBRTC_VIDEO_CODEC_OK;
255       break;
256     default:
257       RTC_NOTREACHED();
258       status = WEBRTC_VIDEO_CODEC_ERROR;
259   }
260 
261   decoder_type_ = DecoderType::kNone;
262   return status;
263 }
264 
PrefersLateDecoding() const265 bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const {
266   return active_decoder().PrefersLateDecoding();
267 }
268 
ImplementationName() const269 const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const {
270   return decoder_type_ == DecoderType::kFallback
271              ? fallback_implementation_name_.c_str()
272              : hw_decoder_->ImplementationName();
273 }
274 
active_decoder() const275 VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const {
276   return decoder_type_ == DecoderType::kFallback ? *fallback_decoder_
277                                                  : *hw_decoder_;
278 }
279 
280 }  // namespace
281 
CreateVideoDecoderSoftwareFallbackWrapper(std::unique_ptr<VideoDecoder> sw_fallback_decoder,std::unique_ptr<VideoDecoder> hw_decoder)282 std::unique_ptr<VideoDecoder> CreateVideoDecoderSoftwareFallbackWrapper(
283     std::unique_ptr<VideoDecoder> sw_fallback_decoder,
284     std::unique_ptr<VideoDecoder> hw_decoder) {
285   return std::make_unique<VideoDecoderSoftwareFallbackWrapper>(
286       std::move(sw_fallback_decoder), std::move(hw_decoder));
287 }
288 
289 }  // namespace webrtc
290