1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/frame_generator.h"
11
12 #include <string.h>
13
14 #include <cstdint>
15 #include <cstdio>
16 #include <memory>
17
18 #include "api/video/i010_buffer.h"
19 #include "api/video/video_rotation.h"
20 #include "common_video/include/video_frame_buffer.h"
21 #include "common_video/libyuv/include/webrtc_libyuv.h"
22 #include "rtc_base/bind.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/keep_ref_until_done.h"
25 #include "test/frame_utils.h"
26
27 namespace webrtc {
28 namespace test {
29 namespace {
30
31 // Helper method for keeping a reference to passed pointers.
KeepBufferRefs(rtc::scoped_refptr<webrtc::VideoFrameBuffer>,rtc::scoped_refptr<webrtc::VideoFrameBuffer>)32 void KeepBufferRefs(rtc::scoped_refptr<webrtc::VideoFrameBuffer>,
33 rtc::scoped_refptr<webrtc::VideoFrameBuffer>) {}
34
35 } // namespace
36
SquareGenerator(int width,int height,OutputType type,int num_squares)37 SquareGenerator::SquareGenerator(int width,
38 int height,
39 OutputType type,
40 int num_squares)
41 : type_(type) {
42 ChangeResolution(width, height);
43 for (int i = 0; i < num_squares; ++i) {
44 squares_.emplace_back(new Square(width, height, i + 1));
45 }
46 }
47
ChangeResolution(size_t width,size_t height)48 void SquareGenerator::ChangeResolution(size_t width, size_t height) {
49 MutexLock lock(&mutex_);
50 width_ = static_cast<int>(width);
51 height_ = static_cast<int>(height);
52 RTC_CHECK(width_ > 0);
53 RTC_CHECK(height_ > 0);
54 }
55
CreateI420Buffer(int width,int height)56 rtc::scoped_refptr<I420Buffer> SquareGenerator::CreateI420Buffer(int width,
57 int height) {
58 rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(width, height));
59 memset(buffer->MutableDataY(), 127, height * buffer->StrideY());
60 memset(buffer->MutableDataU(), 127,
61 buffer->ChromaHeight() * buffer->StrideU());
62 memset(buffer->MutableDataV(), 127,
63 buffer->ChromaHeight() * buffer->StrideV());
64 return buffer;
65 }
66
NextFrame()67 FrameGeneratorInterface::VideoFrameData SquareGenerator::NextFrame() {
68 MutexLock lock(&mutex_);
69
70 rtc::scoped_refptr<VideoFrameBuffer> buffer = nullptr;
71 switch (type_) {
72 case OutputType::kI420:
73 case OutputType::kI010: {
74 buffer = CreateI420Buffer(width_, height_);
75 break;
76 }
77 case OutputType::kI420A: {
78 rtc::scoped_refptr<I420Buffer> yuv_buffer =
79 CreateI420Buffer(width_, height_);
80 rtc::scoped_refptr<I420Buffer> axx_buffer =
81 CreateI420Buffer(width_, height_);
82 buffer = WrapI420ABuffer(
83 yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
84 yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
85 yuv_buffer->DataV(), yuv_buffer->StrideV(), axx_buffer->DataY(),
86 axx_buffer->StrideY(),
87 rtc::Bind(&KeepBufferRefs, yuv_buffer, axx_buffer));
88 break;
89 }
90 default:
91 RTC_NOTREACHED() << "The given output format is not supported.";
92 }
93
94 for (const auto& square : squares_)
95 square->Draw(buffer);
96
97 if (type_ == OutputType::kI010) {
98 buffer = I010Buffer::Copy(*buffer->ToI420());
99 }
100
101 return VideoFrameData(buffer, absl::nullopt);
102 }
103
Square(int width,int height,int seed)104 SquareGenerator::Square::Square(int width, int height, int seed)
105 : random_generator_(seed),
106 x_(random_generator_.Rand(0, width)),
107 y_(random_generator_.Rand(0, height)),
108 length_(random_generator_.Rand(1, width > 4 ? width / 4 : 1)),
109 yuv_y_(random_generator_.Rand(0, 255)),
110 yuv_u_(random_generator_.Rand(0, 255)),
111 yuv_v_(random_generator_.Rand(0, 255)),
112 yuv_a_(random_generator_.Rand(0, 255)) {}
113
Draw(const rtc::scoped_refptr<VideoFrameBuffer> & frame_buffer)114 void SquareGenerator::Square::Draw(
115 const rtc::scoped_refptr<VideoFrameBuffer>& frame_buffer) {
116 RTC_DCHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 ||
117 frame_buffer->type() == VideoFrameBuffer::Type::kI420A);
118 rtc::scoped_refptr<I420BufferInterface> buffer = frame_buffer->ToI420();
119 int length_cap = std::min(buffer->height(), buffer->width()) / 4;
120 int length = std::min(length_, length_cap);
121 x_ = (x_ + random_generator_.Rand(0, 4)) % (buffer->width() - length);
122 y_ = (y_ + random_generator_.Rand(0, 4)) % (buffer->height() - length);
123 for (int y = y_; y < y_ + length; ++y) {
124 uint8_t* pos_y =
125 (const_cast<uint8_t*>(buffer->DataY()) + x_ + y * buffer->StrideY());
126 memset(pos_y, yuv_y_, length);
127 }
128
129 for (int y = y_; y < y_ + length; y = y + 2) {
130 uint8_t* pos_u = (const_cast<uint8_t*>(buffer->DataU()) + x_ / 2 +
131 y / 2 * buffer->StrideU());
132 memset(pos_u, yuv_u_, length / 2);
133 uint8_t* pos_v = (const_cast<uint8_t*>(buffer->DataV()) + x_ / 2 +
134 y / 2 * buffer->StrideV());
135 memset(pos_v, yuv_v_, length / 2);
136 }
137
138 if (frame_buffer->type() == VideoFrameBuffer::Type::kI420)
139 return;
140
141 // Optionally draw on alpha plane if given.
142 const webrtc::I420ABufferInterface* yuva_buffer = frame_buffer->GetI420A();
143 for (int y = y_; y < y_ + length; ++y) {
144 uint8_t* pos_y = (const_cast<uint8_t*>(yuva_buffer->DataA()) + x_ +
145 y * yuva_buffer->StrideA());
146 memset(pos_y, yuv_a_, length);
147 }
148 }
149
YuvFileGenerator(std::vector<FILE * > files,size_t width,size_t height,int frame_repeat_count)150 YuvFileGenerator::YuvFileGenerator(std::vector<FILE*> files,
151 size_t width,
152 size_t height,
153 int frame_repeat_count)
154 : file_index_(0),
155 frame_index_(std::numeric_limits<size_t>::max()),
156 files_(files),
157 width_(width),
158 height_(height),
159 frame_size_(CalcBufferSize(VideoType::kI420,
160 static_cast<int>(width_),
161 static_cast<int>(height_))),
162 frame_buffer_(new uint8_t[frame_size_]),
163 frame_display_count_(frame_repeat_count),
164 current_display_count_(0) {
165 RTC_DCHECK_GT(width, 0);
166 RTC_DCHECK_GT(height, 0);
167 RTC_DCHECK_GT(frame_repeat_count, 0);
168 }
169
~YuvFileGenerator()170 YuvFileGenerator::~YuvFileGenerator() {
171 for (FILE* file : files_)
172 fclose(file);
173 }
174
NextFrame()175 FrameGeneratorInterface::VideoFrameData YuvFileGenerator::NextFrame() {
176 // Empty update by default.
177 VideoFrame::UpdateRect update_rect{0, 0, 0, 0};
178 if (current_display_count_ == 0) {
179 const bool got_new_frame = ReadNextFrame();
180 // Full update on a new frame from file.
181 if (got_new_frame) {
182 update_rect = VideoFrame::UpdateRect{0, 0, static_cast<int>(width_),
183 static_cast<int>(height_)};
184 }
185 }
186 if (++current_display_count_ >= frame_display_count_)
187 current_display_count_ = 0;
188
189 return VideoFrameData(last_read_buffer_, update_rect);
190 }
191
ReadNextFrame()192 bool YuvFileGenerator::ReadNextFrame() {
193 size_t prev_frame_index = frame_index_;
194 size_t prev_file_index = file_index_;
195 last_read_buffer_ = test::ReadI420Buffer(
196 static_cast<int>(width_), static_cast<int>(height_), files_[file_index_]);
197 ++frame_index_;
198 if (!last_read_buffer_) {
199 // No more frames to read in this file, rewind and move to next file.
200 rewind(files_[file_index_]);
201
202 frame_index_ = 0;
203 file_index_ = (file_index_ + 1) % files_.size();
204 last_read_buffer_ =
205 test::ReadI420Buffer(static_cast<int>(width_),
206 static_cast<int>(height_), files_[file_index_]);
207 RTC_CHECK(last_read_buffer_);
208 }
209 return frame_index_ != prev_frame_index || file_index_ != prev_file_index;
210 }
211
SlideGenerator(int width,int height,int frame_repeat_count)212 SlideGenerator::SlideGenerator(int width, int height, int frame_repeat_count)
213 : width_(width),
214 height_(height),
215 frame_display_count_(frame_repeat_count),
216 current_display_count_(0),
217 random_generator_(1234) {
218 RTC_DCHECK_GT(width, 0);
219 RTC_DCHECK_GT(height, 0);
220 RTC_DCHECK_GT(frame_repeat_count, 0);
221 }
222
NextFrame()223 FrameGeneratorInterface::VideoFrameData SlideGenerator::NextFrame() {
224 if (current_display_count_ == 0)
225 GenerateNewFrame();
226 if (++current_display_count_ >= frame_display_count_)
227 current_display_count_ = 0;
228
229 return VideoFrameData(buffer_, absl::nullopt);
230 }
231
GenerateNewFrame()232 void SlideGenerator::GenerateNewFrame() {
233 // The squares should have a varying order of magnitude in order
234 // to simulate variation in the slides' complexity.
235 const int kSquareNum = 1 << (4 + (random_generator_.Rand(0, 3) * 2));
236
237 buffer_ = I420Buffer::Create(width_, height_);
238 memset(buffer_->MutableDataY(), 127, height_ * buffer_->StrideY());
239 memset(buffer_->MutableDataU(), 127,
240 buffer_->ChromaHeight() * buffer_->StrideU());
241 memset(buffer_->MutableDataV(), 127,
242 buffer_->ChromaHeight() * buffer_->StrideV());
243
244 for (int i = 0; i < kSquareNum; ++i) {
245 int length = random_generator_.Rand(1, width_ > 4 ? width_ / 4 : 1);
246 // Limit the length of later squares so that they don't overwrite the
247 // previous ones too much.
248 length = (length * (kSquareNum - i)) / kSquareNum;
249
250 int x = random_generator_.Rand(0, width_ - length);
251 int y = random_generator_.Rand(0, height_ - length);
252 uint8_t yuv_y = random_generator_.Rand(0, 255);
253 uint8_t yuv_u = random_generator_.Rand(0, 255);
254 uint8_t yuv_v = random_generator_.Rand(0, 255);
255
256 for (int yy = y; yy < y + length; ++yy) {
257 uint8_t* pos_y = (buffer_->MutableDataY() + x + yy * buffer_->StrideY());
258 memset(pos_y, yuv_y, length);
259 }
260 for (int yy = y; yy < y + length; yy += 2) {
261 uint8_t* pos_u =
262 (buffer_->MutableDataU() + x / 2 + yy / 2 * buffer_->StrideU());
263 memset(pos_u, yuv_u, length / 2);
264 uint8_t* pos_v =
265 (buffer_->MutableDataV() + x / 2 + yy / 2 * buffer_->StrideV());
266 memset(pos_v, yuv_v, length / 2);
267 }
268 }
269 }
270
ScrollingImageFrameGenerator(Clock * clock,const std::vector<FILE * > & files,size_t source_width,size_t source_height,size_t target_width,size_t target_height,int64_t scroll_time_ms,int64_t pause_time_ms)271 ScrollingImageFrameGenerator::ScrollingImageFrameGenerator(
272 Clock* clock,
273 const std::vector<FILE*>& files,
274 size_t source_width,
275 size_t source_height,
276 size_t target_width,
277 size_t target_height,
278 int64_t scroll_time_ms,
279 int64_t pause_time_ms)
280 : clock_(clock),
281 start_time_(clock->TimeInMilliseconds()),
282 scroll_time_(scroll_time_ms),
283 pause_time_(pause_time_ms),
284 num_frames_(files.size()),
285 target_width_(static_cast<int>(target_width)),
286 target_height_(static_cast<int>(target_height)),
287 current_frame_num_(num_frames_ - 1),
288 prev_frame_not_scrolled_(false),
289 current_source_frame_(nullptr, absl::nullopt),
290 current_frame_(nullptr, absl::nullopt),
291 file_generator_(files, source_width, source_height, 1) {
292 RTC_DCHECK(clock_ != nullptr);
293 RTC_DCHECK_GT(num_frames_, 0);
294 RTC_DCHECK_GE(source_height, target_height);
295 RTC_DCHECK_GE(source_width, target_width);
296 RTC_DCHECK_GE(scroll_time_ms, 0);
297 RTC_DCHECK_GE(pause_time_ms, 0);
298 RTC_DCHECK_GT(scroll_time_ms + pause_time_ms, 0);
299 }
300
301 FrameGeneratorInterface::VideoFrameData
NextFrame()302 ScrollingImageFrameGenerator::NextFrame() {
303 const int64_t kFrameDisplayTime = scroll_time_ + pause_time_;
304 const int64_t now = clock_->TimeInMilliseconds();
305 int64_t ms_since_start = now - start_time_;
306
307 size_t frame_num = (ms_since_start / kFrameDisplayTime) % num_frames_;
308 UpdateSourceFrame(frame_num);
309
310 bool cur_frame_not_scrolled;
311
312 double scroll_factor;
313 int64_t time_into_frame = ms_since_start % kFrameDisplayTime;
314 if (time_into_frame < scroll_time_) {
315 scroll_factor = static_cast<double>(time_into_frame) / scroll_time_;
316 cur_frame_not_scrolled = false;
317 } else {
318 scroll_factor = 1.0;
319 cur_frame_not_scrolled = true;
320 }
321 CropSourceToScrolledImage(scroll_factor);
322
323 bool same_scroll_position =
324 prev_frame_not_scrolled_ && cur_frame_not_scrolled;
325 if (!same_scroll_position) {
326 // If scrolling is not finished yet, force full frame update.
327 current_frame_.update_rect =
328 VideoFrame::UpdateRect{0, 0, target_width_, target_height_};
329 }
330 prev_frame_not_scrolled_ = cur_frame_not_scrolled;
331
332 return current_frame_;
333 }
334
UpdateSourceFrame(size_t frame_num)335 void ScrollingImageFrameGenerator::UpdateSourceFrame(size_t frame_num) {
336 VideoFrame::UpdateRect acc_update{0, 0, 0, 0};
337 while (current_frame_num_ != frame_num) {
338 current_source_frame_ = file_generator_.NextFrame();
339 if (current_source_frame_.update_rect) {
340 acc_update.Union(*current_source_frame_.update_rect);
341 }
342 current_frame_num_ = (current_frame_num_ + 1) % num_frames_;
343 }
344 current_source_frame_.update_rect = acc_update;
345 }
346
CropSourceToScrolledImage(double scroll_factor)347 void ScrollingImageFrameGenerator::CropSourceToScrolledImage(
348 double scroll_factor) {
349 int scroll_margin_x = current_source_frame_.buffer->width() - target_width_;
350 int pixels_scrolled_x =
351 static_cast<int>(scroll_margin_x * scroll_factor + 0.5);
352 int scroll_margin_y = current_source_frame_.buffer->height() - target_height_;
353 int pixels_scrolled_y =
354 static_cast<int>(scroll_margin_y * scroll_factor + 0.5);
355
356 rtc::scoped_refptr<I420BufferInterface> i420_buffer =
357 current_source_frame_.buffer->ToI420();
358 int offset_y =
359 (i420_buffer->StrideY() * pixels_scrolled_y) + pixels_scrolled_x;
360 int offset_u = (i420_buffer->StrideU() * (pixels_scrolled_y / 2)) +
361 (pixels_scrolled_x / 2);
362 int offset_v = (i420_buffer->StrideV() * (pixels_scrolled_y / 2)) +
363 (pixels_scrolled_x / 2);
364
365 VideoFrame::UpdateRect update_rect =
366 current_source_frame_.update_rect->IsEmpty()
367 ? VideoFrame::UpdateRect{0, 0, 0, 0}
368 : VideoFrame::UpdateRect{0, 0, target_width_, target_height_};
369 current_frame_ = VideoFrameData(
370 WrapI420Buffer(target_width_, target_height_,
371 &i420_buffer->DataY()[offset_y], i420_buffer->StrideY(),
372 &i420_buffer->DataU()[offset_u], i420_buffer->StrideU(),
373 &i420_buffer->DataV()[offset_v], i420_buffer->StrideV(),
374 KeepRefUntilDone(i420_buffer)),
375 update_rect);
376 }
377
378 } // namespace test
379 } // namespace webrtc
380