1 /*
2  * libjingle
3  * Copyright 2011 Google Inc.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  *  1. Redistributions of source code must retain the above copyright notice,
9  *     this list of conditions and the following disclaimer.
10  *  2. Redistributions in binary form must reproduce the above copyright notice,
11  *     this list of conditions and the following disclaimer in the documentation
12  *     and/or other materials provided with the distribution.
13  *  3. The name of the author may not be used to endorse or promote products
14  *     derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include "talk/media/base/videoframe.h"
29 
30 #include <string.h>
31 
32 #include "libyuv/compare.h"
33 #include "libyuv/planar_functions.h"
34 #include "libyuv/scale.h"
35 #include "talk/media/base/videocommon.h"
36 #include "webrtc/base/arraysize.h"
37 #include "webrtc/base/checks.h"
38 #include "webrtc/base/logging.h"
39 
40 namespace cricket {
41 
42 // Round to 2 pixels because Chroma channels are half size.
43 #define ROUNDTO2(v) (v & ~1)
44 
Write(rtc::StreamInterface * stream,int * error) const45 rtc::StreamResult VideoFrame::Write(rtc::StreamInterface* stream,
46                                           int* error) const {
47   rtc::StreamResult result = rtc::SR_SUCCESS;
48   const uint8_t* src_y = GetYPlane();
49   const uint8_t* src_u = GetUPlane();
50   const uint8_t* src_v = GetVPlane();
51   if (!src_y || !src_u || !src_v) {
52     return result;  // Nothing to write.
53   }
54   const int32_t y_pitch = GetYPitch();
55   const int32_t u_pitch = GetUPitch();
56   const int32_t v_pitch = GetVPitch();
57   const size_t width = GetWidth();
58   const size_t height = GetHeight();
59   const size_t half_width = (width + 1) >> 1;
60   const size_t half_height = (height + 1) >> 1;
61   // Write Y.
62   for (size_t row = 0; row < height; ++row) {
63     result = stream->Write(src_y + row * y_pitch, width, NULL, error);
64     if (result != rtc::SR_SUCCESS) {
65       return result;
66     }
67   }
68   // Write U.
69   for (size_t row = 0; row < half_height; ++row) {
70     result = stream->Write(src_u + row * u_pitch, half_width, NULL, error);
71     if (result != rtc::SR_SUCCESS) {
72       return result;
73     }
74   }
75   // Write V.
76   for (size_t row = 0; row < half_height; ++row) {
77     result = stream->Write(src_v + row * v_pitch, half_width, NULL, error);
78     if (result != rtc::SR_SUCCESS) {
79       return result;
80     }
81   }
82   return result;
83 }
84 
CopyToBuffer(uint8_t * buffer,size_t size) const85 size_t VideoFrame::CopyToBuffer(uint8_t* buffer, size_t size) const {
86   const size_t y_size = GetHeight() * GetYPitch();
87   const size_t u_size = GetUPitch() * GetChromaHeight();
88   const size_t v_size = GetVPitch() * GetChromaHeight();
89   const size_t needed = y_size + u_size + v_size;
90   if (size < needed)
91     return needed;
92   CopyToPlanes(buffer, buffer + y_size, buffer + y_size + u_size,
93                GetYPitch(), GetUPitch(), GetVPitch());
94   return needed;
95 }
96 
CopyToPlanes(uint8_t * dst_y,uint8_t * dst_u,uint8_t * dst_v,int32_t dst_pitch_y,int32_t dst_pitch_u,int32_t dst_pitch_v) const97 bool VideoFrame::CopyToPlanes(uint8_t* dst_y,
98                               uint8_t* dst_u,
99                               uint8_t* dst_v,
100                               int32_t dst_pitch_y,
101                               int32_t dst_pitch_u,
102                               int32_t dst_pitch_v) const {
103   if (!GetYPlane() || !GetUPlane() || !GetVPlane()) {
104     LOG(LS_ERROR) << "NULL plane pointer.";
105     return false;
106   }
107   int32_t src_width = static_cast<int>(GetWidth());
108   int32_t src_height = static_cast<int>(GetHeight());
109   return libyuv::I420Copy(GetYPlane(), GetYPitch(),
110                           GetUPlane(), GetUPitch(),
111                           GetVPlane(), GetVPitch(),
112                           dst_y, dst_pitch_y,
113                           dst_u, dst_pitch_u,
114                           dst_v, dst_pitch_v,
115                           src_width, src_height) == 0;
116 }
117 
CopyToFrame(VideoFrame * dst) const118 void VideoFrame::CopyToFrame(VideoFrame* dst) const {
119   if (!dst) {
120     LOG(LS_ERROR) << "NULL dst pointer.";
121     return;
122   }
123 
124   CopyToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(),
125                dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch());
126 }
127 
ConvertToRgbBuffer(uint32_t to_fourcc,uint8_t * buffer,size_t size,int stride_rgb) const128 size_t VideoFrame::ConvertToRgbBuffer(uint32_t to_fourcc,
129                                       uint8_t* buffer,
130                                       size_t size,
131                                       int stride_rgb) const {
132   const size_t needed = std::abs(stride_rgb) * GetHeight();
133   if (size < needed) {
134     LOG(LS_WARNING) << "RGB buffer is not large enough";
135     return needed;
136   }
137 
138   if (libyuv::ConvertFromI420(GetYPlane(), GetYPitch(), GetUPlane(),
139                               GetUPitch(), GetVPlane(), GetVPitch(), buffer,
140                               stride_rgb, static_cast<int>(GetWidth()),
141                               static_cast<int>(GetHeight()), to_fourcc)) {
142     LOG(LS_ERROR) << "RGB type not supported: " << to_fourcc;
143     return 0;  // 0 indicates error
144   }
145   return needed;
146 }
147 
148 // TODO(fbarchard): Handle odd width/height with rounding.
StretchToPlanes(uint8_t * dst_y,uint8_t * dst_u,uint8_t * dst_v,int32_t dst_pitch_y,int32_t dst_pitch_u,int32_t dst_pitch_v,size_t width,size_t height,bool interpolate,bool vert_crop) const149 void VideoFrame::StretchToPlanes(uint8_t* dst_y,
150                                  uint8_t* dst_u,
151                                  uint8_t* dst_v,
152                                  int32_t dst_pitch_y,
153                                  int32_t dst_pitch_u,
154                                  int32_t dst_pitch_v,
155                                  size_t width,
156                                  size_t height,
157                                  bool interpolate,
158                                  bool vert_crop) const {
159   if (!GetYPlane() || !GetUPlane() || !GetVPlane()) {
160     LOG(LS_ERROR) << "NULL plane pointer.";
161     return;
162   }
163 
164   size_t src_width = GetWidth();
165   size_t src_height = GetHeight();
166   if (width == src_width && height == src_height) {
167     CopyToPlanes(dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v);
168     return;
169   }
170   const uint8_t* src_y = GetYPlane();
171   const uint8_t* src_u = GetUPlane();
172   const uint8_t* src_v = GetVPlane();
173 
174   if (vert_crop) {
175     // Adjust the input width:height ratio to be the same as the output ratio.
176     if (src_width * height > src_height * width) {
177       // Reduce the input width, but keep size/position aligned for YuvScaler
178       src_width = ROUNDTO2(src_height * width / height);
179       int32_t iwidth_offset = ROUNDTO2((GetWidth() - src_width) / 2);
180       src_y += iwidth_offset;
181       src_u += iwidth_offset / 2;
182       src_v += iwidth_offset / 2;
183     } else if (src_width * height < src_height * width) {
184       // Reduce the input height.
185       src_height = src_width * height / width;
186       int32_t iheight_offset =
187           static_cast<int32_t>((GetHeight() - src_height) >> 2);
188       iheight_offset <<= 1;  // Ensure that iheight_offset is even.
189       src_y += iheight_offset * GetYPitch();
190       src_u += iheight_offset / 2 * GetUPitch();
191       src_v += iheight_offset / 2 * GetVPitch();
192     }
193   }
194 
195   // Scale to the output I420 frame.
196   libyuv::Scale(src_y, src_u, src_v,
197                 GetYPitch(), GetUPitch(), GetVPitch(),
198                 static_cast<int>(src_width), static_cast<int>(src_height),
199                 dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v,
200                 static_cast<int>(width), static_cast<int>(height), interpolate);
201 }
202 
StretchToFrame(VideoFrame * dst,bool interpolate,bool vert_crop) const203 void VideoFrame::StretchToFrame(VideoFrame* dst,
204                                 bool interpolate, bool vert_crop) const {
205   if (!dst) {
206     LOG(LS_ERROR) << "NULL dst pointer.";
207     return;
208   }
209 
210   StretchToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(),
211                   dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch(),
212                   dst->GetWidth(), dst->GetHeight(),
213                   interpolate, vert_crop);
214   dst->SetTimeStamp(GetTimeStamp());
215   // Stretched frame should have the same rotation as the source.
216   dst->SetRotation(GetVideoRotation());
217 }
218 
Stretch(size_t dst_width,size_t dst_height,bool interpolate,bool vert_crop) const219 VideoFrame* VideoFrame::Stretch(size_t dst_width, size_t dst_height,
220                                 bool interpolate, bool vert_crop) const {
221   VideoFrame* dest = CreateEmptyFrame(static_cast<int>(dst_width),
222                                       static_cast<int>(dst_height),
223                                       GetPixelWidth(), GetPixelHeight(),
224                                       GetTimeStamp());
225   if (dest) {
226     StretchToFrame(dest, interpolate, vert_crop);
227   }
228   return dest;
229 }
230 
SetToBlack()231 bool VideoFrame::SetToBlack() {
232   return libyuv::I420Rect(GetYPlane(), GetYPitch(),
233                           GetUPlane(), GetUPitch(),
234                           GetVPlane(), GetVPitch(),
235                           0, 0,
236                           static_cast<int>(GetWidth()),
237                           static_cast<int>(GetHeight()),
238                           16, 128, 128) == 0;
239 }
240 
241 static const size_t kMaxSampleSize = 1000000000u;
242 // Returns whether a sample is valid.
Validate(uint32_t fourcc,int w,int h,const uint8_t * sample,size_t sample_size)243 bool VideoFrame::Validate(uint32_t fourcc,
244                           int w,
245                           int h,
246                           const uint8_t* sample,
247                           size_t sample_size) {
248   if (h < 0) {
249     h = -h;
250   }
251   // 16384 is maximum resolution for VP8 codec.
252   if (w < 1 || w > 16384 || h < 1 || h > 16384) {
253     LOG(LS_ERROR) << "Invalid dimensions: " << w << "x" << h;
254     return false;
255   }
256   uint32_t format = CanonicalFourCC(fourcc);
257   int expected_bpp = 8;
258   switch (format) {
259     case FOURCC_I400:
260     case FOURCC_RGGB:
261     case FOURCC_BGGR:
262     case FOURCC_GRBG:
263     case FOURCC_GBRG:
264       expected_bpp = 8;
265       break;
266     case FOURCC_I420:
267     case FOURCC_I411:
268     case FOURCC_YU12:
269     case FOURCC_YV12:
270     case FOURCC_M420:
271     case FOURCC_NV21:
272     case FOURCC_NV12:
273       expected_bpp = 12;
274       break;
275     case FOURCC_I422:
276     case FOURCC_YV16:
277     case FOURCC_YUY2:
278     case FOURCC_UYVY:
279     case FOURCC_RGBP:
280     case FOURCC_RGBO:
281     case FOURCC_R444:
282       expected_bpp = 16;
283       break;
284     case FOURCC_I444:
285     case FOURCC_YV24:
286     case FOURCC_24BG:
287     case FOURCC_RAW:
288       expected_bpp = 24;
289       break;
290 
291     case FOURCC_ABGR:
292     case FOURCC_BGRA:
293     case FOURCC_ARGB:
294       expected_bpp = 32;
295       break;
296 
297     case FOURCC_MJPG:
298     case FOURCC_H264:
299       expected_bpp = 0;
300       break;
301     default:
302       expected_bpp = 8;  // Expect format is at least 8 bits per pixel.
303       break;
304   }
305   size_t expected_size = (w * expected_bpp + 7) / 8 * h;
306   // For compressed formats, expect 4 bits per 16 x 16 macro.  I420 would be
307   // 6 bits, but grey can be 4 bits.
308   if (expected_bpp == 0) {
309     expected_size = ((w + 15) / 16) * ((h + 15) / 16) * 4 / 8;
310   }
311   if (sample == NULL) {
312     LOG(LS_ERROR) << "NULL sample pointer."
313                   << " format: " << GetFourccName(format)
314                   << " bpp: " << expected_bpp
315                   << " size: " << w << "x" << h
316                   << " expected: " << expected_size
317                   << " " << sample_size;
318     return false;
319   }
320   // TODO(fbarchard): Make function to dump information about frames.
321   uint8_t four_samples[4] = {0, 0, 0, 0};
322   for (size_t i = 0; i < arraysize(four_samples) && i < sample_size; ++i) {
323     four_samples[i] = sample[i];
324   }
325   if (sample_size < expected_size) {
326     LOG(LS_ERROR) << "Size field is too small."
327                   << " format: " << GetFourccName(format)
328                   << " bpp: " << expected_bpp
329                   << " size: " << w << "x" << h
330                   << " " << sample_size
331                   << " expected: " << expected_size
332                   << " sample[0..3]: " << static_cast<int>(four_samples[0])
333                   << ", " << static_cast<int>(four_samples[1])
334                   << ", " << static_cast<int>(four_samples[2])
335                   << ", " << static_cast<int>(four_samples[3]);
336     return false;
337   }
338   if (sample_size > kMaxSampleSize) {
339     LOG(LS_WARNING) << "Size field is invalid."
340                     << " format: " << GetFourccName(format)
341                     << " bpp: " << expected_bpp
342                     << " size: " << w << "x" << h
343                     << " " << sample_size
344                     << " expected: " << 2 * expected_size
345                     << " sample[0..3]: " << static_cast<int>(four_samples[0])
346                     << ", " << static_cast<int>(four_samples[1])
347                     << ", " << static_cast<int>(four_samples[2])
348                     << ", " << static_cast<int>(four_samples[3]);
349     return false;
350   }
351   // Show large size warning once every 100 frames.
352   // TODO(fbarchard): Make frame counter atomic for thread safety.
353   static int large_warn100 = 0;
354   size_t large_expected_size = expected_size * 2;
355   if (expected_bpp >= 8 &&
356       (sample_size > large_expected_size || sample_size > kMaxSampleSize) &&
357       large_warn100 % 100 == 0) {
358     ++large_warn100;
359     LOG(LS_WARNING) << "Size field is too large."
360                     << " format: " << GetFourccName(format)
361                     << " bpp: " << expected_bpp
362                     << " size: " << w << "x" << h
363                     << " bytes: " << sample_size
364                     << " expected: " << large_expected_size
365                     << " sample[0..3]: " << static_cast<int>(four_samples[0])
366                     << ", " << static_cast<int>(four_samples[1])
367                     << ", " << static_cast<int>(four_samples[2])
368                     << ", " << static_cast<int>(four_samples[3]);
369   }
370 
371   // TODO(fbarchard): Add duplicate pixel check.
372   // TODO(fbarchard): Use frame counter atomic for thread safety.
373   static bool valid_once = true;
374   if (valid_once) {
375     valid_once = false;
376     LOG(LS_INFO) << "Validate frame passed."
377                  << " format: " << GetFourccName(format)
378                  << " bpp: " << expected_bpp
379                  << " size: " << w << "x" << h
380                  << " bytes: " << sample_size
381                  << " expected: " << expected_size
382                  << " sample[0..3]: " << static_cast<int>(four_samples[0])
383                  << ", " << static_cast<int>(four_samples[1])
384                  << ", " << static_cast<int>(four_samples[2])
385                  << ", " << static_cast<int>(four_samples[3]);
386   }
387   return true;
388 }
389 
390 }  // namespace cricket
391