1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  *
10  */
11 
12 #ifndef MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
13 #define MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
14 
15 #ifdef RTC_ENABLE_VP9
16 
17 #include <map>
18 #include <memory>
19 #include <string>
20 #include <vector>
21 
22 #include "api/fec_controller_override.h"
23 #include "api/video_codecs/video_encoder.h"
24 #include "media/base/vp9_profile.h"
25 #include "modules/video_coding/codecs/vp9/include/vp9.h"
26 #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
27 #include "modules/video_coding/utility/framerate_controller.h"
28 #include "vpx/vp8cx.h"
29 #include "vpx/vpx_decoder.h"
30 #include "vpx/vpx_encoder.h"
31 
32 namespace webrtc {
33 
34 class VP9EncoderImpl : public VP9Encoder {
35  public:
36   explicit VP9EncoderImpl(const cricket::VideoCodec& codec);
37 
38   ~VP9EncoderImpl() override;
39 
40   void SetFecControllerOverride(
41       FecControllerOverride* fec_controller_override) override;
42 
43   int Release() override;
44 
45   int InitEncode(const VideoCodec* codec_settings,
46                  const Settings& settings) override;
47 
48   int Encode(const VideoFrame& input_image,
49              const std::vector<VideoFrameType>* frame_types) override;
50 
51   int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
52 
53   void SetRates(const RateControlParameters& parameters) override;
54 
55   EncoderInfo GetEncoderInfo() const override;
56 
57  private:
58   // Determine number of encoder threads to use.
59   int NumberOfThreads(int width, int height, int number_of_cores);
60 
61   // Call encoder initialize function and set control settings.
62   int InitAndSetControlSettings(const VideoCodec* inst);
63 
64   void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
65                              absl::optional<int>* spatial_idx,
66                              const vpx_codec_cx_pkt& pkt,
67                              uint32_t timestamp);
68   void FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
69                             const size_t pic_num,
70                             const bool inter_layer_predicted,
71                             CodecSpecificInfoVP9* vp9_info);
72   void UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
73                               const size_t pic_num);
74   vpx_svc_ref_frame_config_t SetReferences(
75       bool is_key_pic,
76       size_t first_active_spatial_layer_id);
77 
78   bool ExplicitlyConfiguredSpatialLayers() const;
79   bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation);
80 
81   virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt);
82 
83   // Callback function for outputting packets per spatial layer.
84   static void EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
85                                                void* user_data);
86 
87   void DeliverBufferedFrame(bool end_of_picture);
88 
89   bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp);
90 
91   // Determine maximum target for Intra frames
92   //
93   // Input:
94   //    - optimal_buffer_size : Optimal buffer size
95   // Return Value             : Max target size for Intra frames represented as
96   //                            percentage of the per frame bandwidth
97   uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
98 
99   size_t SteadyStateSize(int sid, int tid);
100 
101   EncodedImage encoded_image_;
102   CodecSpecificInfo codec_specific_;
103   EncodedImageCallback* encoded_complete_callback_;
104   VideoCodec codec_;
105   const VP9Profile profile_;
106   bool inited_;
107   int64_t timestamp_;
108   int cpu_speed_;
109   uint32_t rc_max_intra_target_;
110   vpx_codec_ctx_t* encoder_;
111   vpx_codec_enc_cfg_t* config_;
112   vpx_image_t* raw_;
113   vpx_svc_extra_cfg_t svc_params_;
114   const VideoFrame* input_image_;
115   GofInfoVP9 gof_;  // Contains each frame's temporal information for
116                     // non-flexible mode.
117   bool force_key_frame_;
118   size_t pics_since_key_;
119   uint8_t num_temporal_layers_;
120   uint8_t num_spatial_layers_;         // Number of configured SLs
121   uint8_t num_active_spatial_layers_;  // Number of actively encoded SLs
122   uint8_t first_active_layer_;
123   bool layer_deactivation_requires_key_frame_;
124   bool is_svc_;
125   InterLayerPredMode inter_layer_pred_;
126   bool external_ref_control_;
127   const bool trusted_rate_controller_;
128   const bool dynamic_rate_settings_;
129   bool layer_buffering_;
130   const bool full_superframe_drop_;
131   vpx_svc_frame_drop_t svc_drop_frame_;
132   bool first_frame_in_picture_;
133   VideoBitrateAllocation current_bitrate_allocation_;
134   bool ss_info_needed_;
135   bool force_all_active_layers_;
136 
137   std::vector<FramerateController> framerate_controller_;
138 
139   // Used for flexible mode.
140   bool is_flexible_mode_;
141   struct RefFrameBuffer {
RefFrameBufferRefFrameBuffer142     RefFrameBuffer(size_t pic_num,
143                    size_t spatial_layer_id,
144                    size_t temporal_layer_id)
145         : pic_num(pic_num),
146           spatial_layer_id(spatial_layer_id),
147           temporal_layer_id(temporal_layer_id) {}
RefFrameBufferRefFrameBuffer148     RefFrameBuffer() {}
149 
150     bool operator==(const RefFrameBuffer& o) {
151       return pic_num == o.pic_num && spatial_layer_id == o.spatial_layer_id &&
152              temporal_layer_id == o.temporal_layer_id;
153     }
154 
155     size_t pic_num = 0;
156     size_t spatial_layer_id = 0;
157     size_t temporal_layer_id = 0;
158   };
159   std::map<size_t, RefFrameBuffer> ref_buf_;
160 
161   // Variable frame-rate related fields and methods.
162   const struct VariableFramerateExperiment {
163     bool enabled;
164     // Framerate is limited to this value in steady state.
165     float framerate_limit;
166     // This qp or below is considered a steady state.
167     int steady_state_qp;
168     // Frames of at least this percentage below ideal for configured bitrate are
169     // considered in a steady state.
170     int steady_state_undershoot_percentage;
171     // Number of consecutive frames with good QP and size required to detect
172     // the steady state.
173     int frames_before_steady_state;
174   } variable_framerate_experiment_;
175   static VariableFramerateExperiment ParseVariableFramerateConfig(
176       std::string group_name);
177   FramerateController variable_framerate_controller_;
178 
179   const struct QualityScalerExperiment {
180     int low_qp;
181     int high_qp;
182     bool enabled;
183   } quality_scaler_experiment_;
184   static QualityScalerExperiment ParseQualityScalerConfig(
185       std::string group_name);
186 
187   int num_steady_state_frames_;
188   // Only set config when this flag is set.
189   bool config_changed_;
190 };
191 
192 class VP9DecoderImpl : public VP9Decoder {
193  public:
194   VP9DecoderImpl();
195 
196   virtual ~VP9DecoderImpl();
197 
198   int InitDecode(const VideoCodec* inst, int number_of_cores) override;
199 
200   int Decode(const EncodedImage& input_image,
201              bool missing_frames,
202              int64_t /*render_time_ms*/) override;
203 
204   int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
205 
206   int Release() override;
207 
208   const char* ImplementationName() const override;
209 
210  private:
211   int ReturnFrame(const vpx_image_t* img,
212                   uint32_t timestamp,
213                   int qp,
214                   const webrtc::ColorSpace* explicit_color_space);
215 
216   // Memory pool used to share buffers between libvpx and webrtc.
217   Vp9FrameBufferPool frame_buffer_pool_;
218   DecodedImageCallback* decode_complete_callback_;
219   bool inited_;
220   vpx_codec_ctx_t* decoder_;
221   bool key_frame_required_;
222   VideoCodec current_codec_;
223   int num_cores_;
224 };
225 }  // namespace webrtc
226 
227 #endif  // RTC_ENABLE_VP9
228 
229 #endif  // MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
230