1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 12 #define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 13 14 #include <list> 15 #include <memory> 16 #include <utility> 17 #include <vector> 18 19 #include "absl/strings/string_view.h" 20 #include "api/test/video_quality_analyzer_interface.h" 21 #include "api/video/video_frame.h" 22 #include "api/video_codecs/sdp_video_format.h" 23 #include "api/video_codecs/video_codec.h" 24 #include "api/video_codecs/video_encoder.h" 25 #include "api/video_codecs/video_encoder_factory.h" 26 #include "rtc_base/synchronization/mutex.h" 27 #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" 28 #include "test/pc/e2e/analyzer/video/id_generator.h" 29 30 namespace webrtc { 31 namespace webrtc_pc_e2e { 32 33 // Tells QualityAnalyzingVideoEncoder that it shouldn't mark any spatial stream 34 // as to be discarded. In such case the top stream will be passed to 35 // VideoQualityAnalyzerInterface as a reference. 36 constexpr int kAnalyzeAnySpatialStream = -1; 37 38 // QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject 39 // VideoQualityAnalyzerInterface before and after encoder. 40 // 41 // QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. 42 // It registers its own EncodedImageCallback in the origin encoder and will 43 // store user specified callback inside itself. 44 // 45 // When Encode(...) will be invoked, quality encoder first calls video quality 46 // analyzer with original frame, then encodes frame with original encoder. 47 // 48 // When origin encoder encodes the image it will call quality encoder's special 49 // callback, where video analyzer will be called again and then frame id will be 50 // injected into EncodedImage with passed EncodedImageDataInjector. Then new 51 // EncodedImage will be passed to origin callback, provided by user. 52 // 53 // Quality encoder registers its own callback in origin encoder, at the same 54 // time the user registers their callback in quality encoder. 55 class QualityAnalyzingVideoEncoder : public VideoEncoder, 56 public EncodedImageCallback { 57 public: 58 // Creates analyzing encoder. |id| is unique coding entity id, that will 59 // be used to distinguish all encoders and decoders inside 60 // EncodedImageDataInjector and EncodedImageIdExtracor. 61 QualityAnalyzingVideoEncoder( 62 int id, 63 absl::string_view peer_name, 64 std::unique_ptr<VideoEncoder> delegate, 65 double bitrate_multiplier, 66 std::map<std::string, absl::optional<int>> stream_required_spatial_index, 67 EncodedImageDataInjector* injector, 68 VideoQualityAnalyzerInterface* analyzer); 69 ~QualityAnalyzingVideoEncoder() override; 70 71 // Methods of VideoEncoder interface. 72 void SetFecControllerOverride( 73 FecControllerOverride* fec_controller_override) override; 74 int32_t InitEncode(const VideoCodec* codec_settings, 75 const Settings& settings) override; 76 int32_t RegisterEncodeCompleteCallback( 77 EncodedImageCallback* callback) override; 78 int32_t Release() override; 79 int32_t Encode(const VideoFrame& frame, 80 const std::vector<VideoFrameType>* frame_types) override; 81 void SetRates(const VideoEncoder::RateControlParameters& parameters) override; 82 EncoderInfo GetEncoderInfo() const override; 83 84 // Methods of EncodedImageCallback interface. 85 EncodedImageCallback::Result OnEncodedImage( 86 const EncodedImage& encoded_image, 87 const CodecSpecificInfo* codec_specific_info, 88 const RTPFragmentationHeader* fragmentation) override; 89 void OnDroppedFrame(DropReason reason) override; 90 91 private: 92 enum SimulcastMode { 93 // In this mode encoder assumes not more than 1 encoded image per video 94 // frame 95 kNormal, 96 97 // Next modes are to test video conference behavior. For conference sender 98 // will send multiple spatial layers/simulcast streams for single video 99 // track and there is some Selective Forwarding Unit (SFU), that forwards 100 // only best one, that will pass through downlink to the receiver. 101 // 102 // Here this behavior will be partly emulated. Sender will send all spatial 103 // layers/simulcast streams and then some of them will be filtered out on 104 // the receiver side. During test setup user can specify which spatial 105 // layer/simulcast stream is required, what will simulated which spatial 106 // layer/simulcast stream will be chosen by SFU in the real world. Then 107 // sender will mark encoded images for all spatial layers above required or 108 // all simulcast streams except required as to be discarded and on receiver 109 // side they will be discarded in quality analyzing decoder and won't be 110 // passed into delegate decoder. 111 // 112 // If the sender for some reasons won't send specified spatial layer, then 113 // receiver still will fall back on lower spatial layers. But for simulcast 114 // streams if required one won't be sent, receiver will assume all frames 115 // in that period as dropped and will experience video freeze. 116 // 117 // Test based on this simulation will be used to evaluate video quality 118 // of concrete spatial layers/simulcast streams and also check distribution 119 // of bandwidth between spatial layers/simulcast streams by BWE. 120 121 // In this mode encoder assumes that for each frame simulcast encoded 122 // images will be produced. So all simulcast streams except required will 123 // be marked as to be discarded in decoder and won't reach video quality 124 // analyzer. 125 kSimulcast, 126 // In this mode encoder assumes that for each frame encoded images for 127 // different spatial layers will be produced. So all spatial layers above 128 // required will be marked to be discarded in decoder and won't reach 129 // video quality analyzer. 130 kSVC, 131 // In this mode encoder assumes that for each frame encoded images for 132 // different spatial layers will be produced. Compared to kSVC mode 133 // spatial layers that are above required will be marked to be discarded 134 // only for key frames and for regular frames all except required spatial 135 // layer will be marked as to be discarded in decoder and won't reach video 136 // quality analyzer. 137 kKSVC 138 }; 139 140 bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) 141 RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); 142 143 const int id_; 144 const std::string peer_name_; 145 std::unique_ptr<VideoEncoder> delegate_; 146 const double bitrate_multiplier_; 147 // Contains mapping from stream label to optional spatial index. 148 // If we have stream label "Foo" and mapping contains 149 // 1. |absl::nullopt| means "Foo" isn't simulcast/SVC stream 150 // 2. |kAnalyzeAnySpatialStream| means all simulcast/SVC streams are required 151 // 3. Concrete value means that particular simulcast/SVC stream have to be 152 // analyzed. 153 std::map<std::string, absl::optional<int>> stream_required_spatial_index_; 154 EncodedImageDataInjector* const injector_; 155 VideoQualityAnalyzerInterface* const analyzer_; 156 157 // VideoEncoder interface assumes async delivery of encoded images. 158 // This lock is used to protect shared state, that have to be propagated 159 // from received VideoFrame to resulted EncodedImage. 160 Mutex lock_; 161 162 VideoCodec codec_settings_; 163 SimulcastMode mode_ RTC_GUARDED_BY(lock_); 164 EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); 165 std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_ 166 RTC_GUARDED_BY(lock_); 167 VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_); 168 }; 169 170 // Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by 171 // specified factory as delegates. Forwards all other calls to specified 172 // factory. 173 class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { 174 public: 175 QualityAnalyzingVideoEncoderFactory( 176 absl::string_view peer_name, 177 std::unique_ptr<VideoEncoderFactory> delegate, 178 double bitrate_multiplier, 179 std::map<std::string, absl::optional<int>> stream_required_spatial_index, 180 IdGenerator<int>* id_generator, 181 EncodedImageDataInjector* injector, 182 VideoQualityAnalyzerInterface* analyzer); 183 ~QualityAnalyzingVideoEncoderFactory() override; 184 185 // Methods of VideoEncoderFactory interface. 186 std::vector<SdpVideoFormat> GetSupportedFormats() const override; 187 VideoEncoderFactory::CodecInfo QueryVideoEncoder( 188 const SdpVideoFormat& format) const override; 189 std::unique_ptr<VideoEncoder> CreateVideoEncoder( 190 const SdpVideoFormat& format) override; 191 192 private: 193 const std::string peer_name_; 194 std::unique_ptr<VideoEncoderFactory> delegate_; 195 const double bitrate_multiplier_; 196 std::map<std::string, absl::optional<int>> stream_required_spatial_index_; 197 IdGenerator<int>* const id_generator_; 198 EncodedImageDataInjector* const injector_; 199 VideoQualityAnalyzerInterface* const analyzer_; 200 }; 201 202 } // namespace webrtc_pc_e2e 203 } // namespace webrtc 204 205 #endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 206