1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
12 
13 #include <assert.h>
14 #include <string.h>
15 
16 #include <limits>
17 #include <vector>
18 
19 #include "webrtc/system_wrappers/include/cpu_info.h"
20 
21 namespace webrtc {
22 namespace test {
23 
TestConfig()24 TestConfig::TestConfig()
25     : name(""),
26       description(""),
27       test_number(0),
28       input_filename(""),
29       output_filename(""),
30       output_dir("out"),
31       networking_config(),
32       exclude_frame_types(kExcludeOnlyFirstKeyFrame),
33       frame_length_in_bytes(0),
34       use_single_core(false),
35       keyframe_interval(0),
36       codec_settings(NULL),
37       verbose(true) {}
38 
~TestConfig()39 TestConfig::~TestConfig() {}
40 
VideoProcessorImpl(webrtc::VideoEncoder * encoder,webrtc::VideoDecoder * decoder,FrameReader * frame_reader,FrameWriter * frame_writer,PacketManipulator * packet_manipulator,const TestConfig & config,Stats * stats)41 VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder,
42                                        webrtc::VideoDecoder* decoder,
43                                        FrameReader* frame_reader,
44                                        FrameWriter* frame_writer,
45                                        PacketManipulator* packet_manipulator,
46                                        const TestConfig& config,
47                                        Stats* stats)
48     : encoder_(encoder),
49       decoder_(decoder),
50       frame_reader_(frame_reader),
51       frame_writer_(frame_writer),
52       packet_manipulator_(packet_manipulator),
53       config_(config),
54       stats_(stats),
55       encode_callback_(NULL),
56       decode_callback_(NULL),
57       source_buffer_(NULL),
58       first_key_frame_has_been_excluded_(false),
59       last_frame_missing_(false),
60       initialized_(false),
61       encoded_frame_size_(0),
62       encoded_frame_type_(kVideoFrameKey),
63       prev_time_stamp_(0),
64       num_dropped_frames_(0),
65       num_spatial_resizes_(0),
66       last_encoder_frame_width_(0),
67       last_encoder_frame_height_(0),
68       scaler_() {
69   assert(encoder);
70   assert(decoder);
71   assert(frame_reader);
72   assert(frame_writer);
73   assert(packet_manipulator);
74   assert(stats);
75 }
76 
Init()77 bool VideoProcessorImpl::Init() {
78   // Calculate a factor used for bit rate calculations:
79   bit_rate_factor_ = config_.codec_settings->maxFramerate * 0.001 * 8;  // bits
80 
81   // Initialize data structures used by the encoder/decoder APIs
82   size_t frame_length_in_bytes = frame_reader_->FrameLength();
83   source_buffer_ = new uint8_t[frame_length_in_bytes];
84   last_successful_frame_buffer_ = new uint8_t[frame_length_in_bytes];
85   // Set fixed properties common for all frames.
86   // To keep track of spatial resize actions by encoder.
87   last_encoder_frame_width_ = config_.codec_settings->width;
88   last_encoder_frame_height_ = config_.codec_settings->height;
89 
90   // Setup required callbacks for the encoder/decoder:
91   encode_callback_ = new VideoProcessorEncodeCompleteCallback(this);
92   decode_callback_ = new VideoProcessorDecodeCompleteCallback(this);
93   int32_t register_result =
94       encoder_->RegisterEncodeCompleteCallback(encode_callback_);
95   if (register_result != WEBRTC_VIDEO_CODEC_OK) {
96     fprintf(stderr,
97             "Failed to register encode complete callback, return code: "
98             "%d\n",
99             register_result);
100     return false;
101   }
102   register_result = decoder_->RegisterDecodeCompleteCallback(decode_callback_);
103   if (register_result != WEBRTC_VIDEO_CODEC_OK) {
104     fprintf(stderr,
105             "Failed to register decode complete callback, return code: "
106             "%d\n",
107             register_result);
108     return false;
109   }
110   // Init the encoder and decoder
111   uint32_t nbr_of_cores = 1;
112   if (!config_.use_single_core) {
113     nbr_of_cores = CpuInfo::DetectNumberOfCores();
114   }
115   int32_t init_result =
116       encoder_->InitEncode(config_.codec_settings, nbr_of_cores,
117                            config_.networking_config.max_payload_size_in_bytes);
118   if (init_result != WEBRTC_VIDEO_CODEC_OK) {
119     fprintf(stderr, "Failed to initialize VideoEncoder, return code: %d\n",
120             init_result);
121     return false;
122   }
123   init_result = decoder_->InitDecode(config_.codec_settings, nbr_of_cores);
124   if (init_result != WEBRTC_VIDEO_CODEC_OK) {
125     fprintf(stderr, "Failed to initialize VideoDecoder, return code: %d\n",
126             init_result);
127     return false;
128   }
129 
130   if (config_.verbose) {
131     printf("Video Processor:\n");
132     printf("  #CPU cores used  : %d\n", nbr_of_cores);
133     printf("  Total # of frames: %d\n", frame_reader_->NumberOfFrames());
134     printf("  Codec settings:\n");
135     printf("    Start bitrate  : %d kbps\n",
136            config_.codec_settings->startBitrate);
137     printf("    Width          : %d\n", config_.codec_settings->width);
138     printf("    Height         : %d\n", config_.codec_settings->height);
139   }
140   initialized_ = true;
141   return true;
142 }
143 
~VideoProcessorImpl()144 VideoProcessorImpl::~VideoProcessorImpl() {
145   delete[] source_buffer_;
146   delete[] last_successful_frame_buffer_;
147   encoder_->RegisterEncodeCompleteCallback(NULL);
148   delete encode_callback_;
149   decoder_->RegisterDecodeCompleteCallback(NULL);
150   delete decode_callback_;
151 }
152 
SetRates(int bit_rate,int frame_rate)153 void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) {
154   int set_rates_result = encoder_->SetRates(bit_rate, frame_rate);
155   assert(set_rates_result >= 0);
156   if (set_rates_result < 0) {
157     fprintf(stderr,
158             "Failed to update encoder with new rate %d, "
159             "return code: %d\n",
160             bit_rate, set_rates_result);
161   }
162   num_dropped_frames_ = 0;
163   num_spatial_resizes_ = 0;
164 }
165 
EncodedFrameSize()166 size_t VideoProcessorImpl::EncodedFrameSize() {
167   return encoded_frame_size_;
168 }
169 
EncodedFrameType()170 FrameType VideoProcessorImpl::EncodedFrameType() {
171   return encoded_frame_type_;
172 }
173 
NumberDroppedFrames()174 int VideoProcessorImpl::NumberDroppedFrames() {
175   return num_dropped_frames_;
176 }
177 
NumberSpatialResizes()178 int VideoProcessorImpl::NumberSpatialResizes() {
179   return num_spatial_resizes_;
180 }
181 
ProcessFrame(int frame_number)182 bool VideoProcessorImpl::ProcessFrame(int frame_number) {
183   assert(frame_number >= 0);
184   if (!initialized_) {
185     fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n");
186     return false;
187   }
188   // |prev_time_stamp_| is used for getting number of dropped frames.
189   if (frame_number == 0) {
190     prev_time_stamp_ = -1;
191   }
192   if (frame_reader_->ReadFrame(source_buffer_)) {
193     // Copy the source frame to the newly read frame data.
194     source_frame_.CreateFrame(source_buffer_, config_.codec_settings->width,
195                               config_.codec_settings->height, kVideoRotation_0);
196 
197     // Ensure we have a new statistics data object we can fill:
198     FrameStatistic& stat = stats_->NewFrame(frame_number);
199 
200     encode_start_ = TickTime::Now();
201     // Use the frame number as "timestamp" to identify frames
202     source_frame_.set_timestamp(frame_number);
203 
204     // Decide if we're going to force a keyframe:
205     std::vector<FrameType> frame_types(1, kVideoFrameDelta);
206     if (config_.keyframe_interval > 0 &&
207         frame_number % config_.keyframe_interval == 0) {
208       frame_types[0] = kVideoFrameKey;
209     }
210 
211     // For dropped frames, we regard them as zero size encoded frames.
212     encoded_frame_size_ = 0;
213     encoded_frame_type_ = kVideoFrameDelta;
214 
215     int32_t encode_result = encoder_->Encode(source_frame_, NULL, &frame_types);
216 
217     if (encode_result != WEBRTC_VIDEO_CODEC_OK) {
218       fprintf(stderr, "Failed to encode frame %d, return code: %d\n",
219               frame_number, encode_result);
220     }
221     stat.encode_return_code = encode_result;
222     return true;
223   } else {
224     return false;  // we've reached the last frame
225   }
226 }
227 
FrameEncoded(const EncodedImage & encoded_image)228 void VideoProcessorImpl::FrameEncoded(const EncodedImage& encoded_image) {
229   // Timestamp is frame number, so this gives us #dropped frames.
230   int num_dropped_from_prev_encode =
231       encoded_image._timeStamp - prev_time_stamp_ - 1;
232   num_dropped_frames_ += num_dropped_from_prev_encode;
233   prev_time_stamp_ = encoded_image._timeStamp;
234   if (num_dropped_from_prev_encode > 0) {
235     // For dropped frames, we write out the last decoded frame to avoid getting
236     // out of sync for the computation of PSNR and SSIM.
237     for (int i = 0; i < num_dropped_from_prev_encode; i++) {
238       frame_writer_->WriteFrame(last_successful_frame_buffer_);
239     }
240   }
241   // Frame is not dropped, so update the encoded frame size
242   // (encoder callback is only called for non-zero length frames).
243   encoded_frame_size_ = encoded_image._length;
244 
245   encoded_frame_type_ = encoded_image._frameType;
246 
247   TickTime encode_stop = TickTime::Now();
248   int frame_number = encoded_image._timeStamp;
249   FrameStatistic& stat = stats_->stats_[frame_number];
250   stat.encode_time_in_us =
251       GetElapsedTimeMicroseconds(encode_start_, encode_stop);
252   stat.encoding_successful = true;
253   stat.encoded_frame_length_in_bytes = encoded_image._length;
254   stat.frame_number = encoded_image._timeStamp;
255   stat.frame_type = encoded_image._frameType;
256   stat.bit_rate_in_kbps = encoded_image._length * bit_rate_factor_;
257   stat.total_packets =
258       encoded_image._length / config_.networking_config.packet_size_in_bytes +
259       1;
260 
261   // Perform packet loss if criteria is fullfilled:
262   bool exclude_this_frame = false;
263   // Only keyframes can be excluded
264   if (encoded_image._frameType == kVideoFrameKey) {
265     switch (config_.exclude_frame_types) {
266       case kExcludeOnlyFirstKeyFrame:
267         if (!first_key_frame_has_been_excluded_) {
268           first_key_frame_has_been_excluded_ = true;
269           exclude_this_frame = true;
270         }
271         break;
272       case kExcludeAllKeyFrames:
273         exclude_this_frame = true;
274         break;
275       default:
276         assert(false);
277     }
278   }
279   rtc::scoped_ptr<uint8_t[]> copied_buffer(new uint8_t[encoded_image._length]);
280   memcpy(copied_buffer.get(), encoded_image._buffer, encoded_image._length);
281   EncodedImage copied_image;
282   memcpy(&copied_image, &encoded_image, sizeof(copied_image));
283   copied_image._size = copied_image._length;
284   copied_image._buffer = copied_buffer.get();
285   if (!exclude_this_frame) {
286     stat.packets_dropped =
287         packet_manipulator_->ManipulatePackets(&copied_image);
288   }
289 
290   // Keep track of if frames are lost due to packet loss so we can tell
291   // this to the encoder (this is handled by the RTP logic in the full stack)
292   decode_start_ = TickTime::Now();
293   // TODO(kjellander): Pass fragmentation header to the decoder when
294   // CL 172001 has been submitted and PacketManipulator supports this.
295   int32_t decode_result =
296       decoder_->Decode(copied_image, last_frame_missing_, NULL);
297   stat.decode_return_code = decode_result;
298   if (decode_result != WEBRTC_VIDEO_CODEC_OK) {
299     // Write the last successful frame the output file to avoid getting it out
300     // of sync with the source file for SSIM and PSNR comparisons:
301     frame_writer_->WriteFrame(last_successful_frame_buffer_);
302   }
303   // save status for losses so we can inform the decoder for the next frame:
304   last_frame_missing_ = copied_image._length == 0;
305 }
306 
FrameDecoded(const VideoFrame & image)307 void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) {
308   TickTime decode_stop = TickTime::Now();
309   int frame_number = image.timestamp();
310   // Report stats
311   FrameStatistic& stat = stats_->stats_[frame_number];
312   stat.decode_time_in_us =
313       GetElapsedTimeMicroseconds(decode_start_, decode_stop);
314   stat.decoding_successful = true;
315 
316   // Check for resize action (either down or up):
317   if (static_cast<int>(image.width()) != last_encoder_frame_width_ ||
318       static_cast<int>(image.height()) != last_encoder_frame_height_) {
319     ++num_spatial_resizes_;
320     last_encoder_frame_width_ = image.width();
321     last_encoder_frame_height_ = image.height();
322   }
323   // Check if codec size is different from native/original size, and if so,
324   // upsample back to original size: needed for PSNR and SSIM computations.
325   if (image.width() != config_.codec_settings->width ||
326       image.height() != config_.codec_settings->height) {
327     VideoFrame up_image;
328     int ret_val = scaler_.Set(
329         image.width(), image.height(), config_.codec_settings->width,
330         config_.codec_settings->height, kI420, kI420, kScaleBilinear);
331     assert(ret_val >= 0);
332     if (ret_val < 0) {
333       fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n",
334               frame_number, ret_val);
335     }
336     ret_val = scaler_.Scale(image, &up_image);
337     assert(ret_val >= 0);
338     if (ret_val < 0) {
339       fprintf(stderr, "Failed to scale frame: %d, return code: %d\n",
340               frame_number, ret_val);
341     }
342     // TODO(mikhal): Extracting the buffer for now - need to update test.
343     size_t length = CalcBufferSize(kI420, up_image.width(), up_image.height());
344     rtc::scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]);
345     int extracted_length = ExtractBuffer(up_image, length, image_buffer.get());
346     assert(extracted_length > 0);
347     // Update our copy of the last successful frame:
348     memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length);
349     bool write_success = frame_writer_->WriteFrame(image_buffer.get());
350     assert(write_success);
351     if (!write_success) {
352       fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
353     }
354   } else {  // No resize.
355     // Update our copy of the last successful frame:
356     // TODO(mikhal): Add as a member function, so won't be allocated per frame.
357     size_t length = CalcBufferSize(kI420, image.width(), image.height());
358     rtc::scoped_ptr<uint8_t[]> image_buffer(new uint8_t[length]);
359     int extracted_length = ExtractBuffer(image, length, image_buffer.get());
360     assert(extracted_length > 0);
361     memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length);
362 
363     bool write_success = frame_writer_->WriteFrame(image_buffer.get());
364     assert(write_success);
365     if (!write_success) {
366       fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
367     }
368   }
369 }
370 
GetElapsedTimeMicroseconds(const webrtc::TickTime & start,const webrtc::TickTime & stop)371 int VideoProcessorImpl::GetElapsedTimeMicroseconds(
372     const webrtc::TickTime& start,
373     const webrtc::TickTime& stop) {
374   uint64_t encode_time = (stop - start).Microseconds();
375   assert(encode_time <
376          static_cast<unsigned int>(std::numeric_limits<int>::max()));
377   return static_cast<int>(encode_time);
378 }
379 
ExcludeFrameTypesToStr(ExcludeFrameTypes e)380 const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) {
381   switch (e) {
382     case kExcludeOnlyFirstKeyFrame:
383       return "ExcludeOnlyFirstKeyFrame";
384     case kExcludeAllKeyFrames:
385       return "ExcludeAllKeyFrames";
386     default:
387       assert(false);
388       return "Unknown";
389   }
390 }
391 
VideoCodecTypeToStr(webrtc::VideoCodecType e)392 const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) {
393   switch (e) {
394     case kVideoCodecVP8:
395       return "VP8";
396     case kVideoCodecI420:
397       return "I420";
398     case kVideoCodecRED:
399       return "RED";
400     case kVideoCodecULPFEC:
401       return "ULPFEC";
402     case kVideoCodecUnknown:
403       return "Unknown";
404     default:
405       assert(false);
406       return "Unknown";
407   }
408 }
409 
410 // Callbacks
Encoded(const EncodedImage & encoded_image,const webrtc::CodecSpecificInfo * codec_specific_info,const webrtc::RTPFragmentationHeader * fragmentation)411 int32_t VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded(
412     const EncodedImage& encoded_image,
413     const webrtc::CodecSpecificInfo* codec_specific_info,
414     const webrtc::RTPFragmentationHeader* fragmentation) {
415   video_processor_->FrameEncoded(encoded_image);  // Forward to parent class.
416   return 0;
417 }
Decoded(VideoFrame & image)418 int32_t VideoProcessorImpl::VideoProcessorDecodeCompleteCallback::Decoded(
419     VideoFrame& image) {
420   video_processor_->FrameDecoded(image);  // forward to parent class
421   return 0;
422 }
423 
424 }  // namespace test
425 }  // namespace webrtc
426