1/*
2 *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import <Foundation/Foundation.h>
12#import <XCTest/XCTest.h>
13
14#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
15
16#import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h"
17#import "base/RTCVideoFrame.h"
18#import "base/RTCVideoFrameBuffer.h"
19#import "frame_buffer_helpers.h"
20
21#include "common_video/libyuv/include/webrtc_libyuv.h"
22#include "third_party/libyuv/include/libyuv.h"
23
24@interface RTCCVPixelBufferTests : XCTestCase
25@end
26
27@implementation RTCCVPixelBufferTests {
28}
29
30- (void)testRequiresCroppingNoCrop {
31  CVPixelBufferRef pixelBufferRef = NULL;
32  CVPixelBufferCreate(
33      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
34  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
35      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
36
37  XCTAssertFalse([buffer requiresCropping]);
38
39  CVBufferRelease(pixelBufferRef);
40}
41
42- (void)testRequiresCroppingWithCrop {
43  CVPixelBufferRef pixelBufferRef = NULL;
44  CVPixelBufferCreate(
45      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
46  RTC_OBJC_TYPE(RTCCVPixelBuffer) *croppedBuffer =
47      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef
48                                                      adaptedWidth:720
49                                                     adaptedHeight:1280
50                                                         cropWidth:360
51                                                        cropHeight:640
52                                                             cropX:100
53                                                             cropY:100];
54
55  XCTAssertTrue([croppedBuffer requiresCropping]);
56
57  CVBufferRelease(pixelBufferRef);
58}
59
60- (void)testRequiresScalingNoScale {
61  CVPixelBufferRef pixelBufferRef = NULL;
62  CVPixelBufferCreate(
63      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
64
65  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
66      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
67  XCTAssertFalse([buffer requiresScalingToWidth:720 height:1280]);
68
69  CVBufferRelease(pixelBufferRef);
70}
71
72- (void)testRequiresScalingWithScale {
73  CVPixelBufferRef pixelBufferRef = NULL;
74  CVPixelBufferCreate(
75      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
76
77  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
78      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
79  XCTAssertTrue([buffer requiresScalingToWidth:360 height:640]);
80
81  CVBufferRelease(pixelBufferRef);
82}
83
84- (void)testRequiresScalingWithScaleAndMatchingCrop {
85  CVPixelBufferRef pixelBufferRef = NULL;
86  CVPixelBufferCreate(
87      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
88
89  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
90      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef
91                                                      adaptedWidth:720
92                                                     adaptedHeight:1280
93                                                         cropWidth:360
94                                                        cropHeight:640
95                                                             cropX:100
96                                                             cropY:100];
97  XCTAssertFalse([buffer requiresScalingToWidth:360 height:640]);
98
99  CVBufferRelease(pixelBufferRef);
100}
101
102- (void)testBufferSize_NV12 {
103  CVPixelBufferRef pixelBufferRef = NULL;
104  CVPixelBufferCreate(
105      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
106
107  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
108      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
109  XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 576000);
110
111  CVBufferRelease(pixelBufferRef);
112}
113
114- (void)testBufferSize_RGB {
115  CVPixelBufferRef pixelBufferRef = NULL;
116  CVPixelBufferCreate(NULL, 720, 1280, kCVPixelFormatType_32BGRA, NULL, &pixelBufferRef);
117
118  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
119      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
120  XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 0);
121
122  CVBufferRelease(pixelBufferRef);
123}
124
125- (void)testCropAndScale_NV12 {
126  [self cropAndScaleTestWithNV12];
127}
128
129- (void)testCropAndScaleNoOp_NV12 {
130  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
131                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
132                                 outputSize:CGSizeMake(720, 1280)];
133}
134
135- (void)testCropAndScale_NV12FullToVideo {
136  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
137                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
138}
139
140- (void)testCropAndScaleZeroSizeFrame_NV12 {
141  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
142                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
143                                 outputSize:CGSizeMake(0, 0)];
144}
145
146- (void)testCropAndScaleToSmallFormat_NV12 {
147  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
148                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
149                                 outputSize:CGSizeMake(148, 320)];
150}
151
152- (void)testCropAndScaleToOddFormat_NV12 {
153  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
154                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
155                                 outputSize:CGSizeMake(361, 640)];
156}
157
158- (void)testCropAndScale_32BGRA {
159  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32BGRA];
160}
161
162- (void)testCropAndScale_32ARGB {
163  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB];
164}
165
166- (void)testCropAndScaleWithSmallCropInfo_32ARGB {
167  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:2 cropY:3];
168}
169
170- (void)testCropAndScaleWithLargeCropInfo_32ARGB {
171  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:200 cropY:300];
172}
173
174- (void)testToI420_NV12 {
175  [self toI420WithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
176}
177
178- (void)testToI420_32BGRA {
179  [self toI420WithPixelFormat:kCVPixelFormatType_32BGRA];
180}
181
182- (void)testToI420_32ARGB {
183  [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB];
184}
185
186#pragma mark - Shared test code
187
188- (void)cropAndScaleTestWithNV12 {
189  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
190                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
191}
192
193- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat {
194  [self cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat
195                               outputFormat:(OSType)outputFormat
196                                 outputSize:CGSizeMake(360, 640)];
197}
198
199- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat
200                               outputFormat:(OSType)outputFormat
201                                 outputSize:(CGSize)outputSize {
202  CVPixelBufferRef pixelBufferRef = NULL;
203  CVPixelBufferCreate(NULL, 720, 1280, inputFormat, NULL, &pixelBufferRef);
204
205  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(720, 1280);
206  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
207
208  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
209      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
210  XCTAssertEqual(buffer.width, 720);
211  XCTAssertEqual(buffer.height, 1280);
212
213  CVPixelBufferRef outputPixelBufferRef = NULL;
214  CVPixelBufferCreate(
215      NULL, outputSize.width, outputSize.height, outputFormat, NULL, &outputPixelBufferRef);
216
217  std::vector<uint8_t> frameScaleBuffer;
218  if ([buffer requiresScalingToWidth:outputSize.width height:outputSize.height]) {
219    int size =
220        [buffer bufferSizeForCroppingAndScalingToWidth:outputSize.width height:outputSize.height];
221    frameScaleBuffer.resize(size);
222  } else {
223    frameScaleBuffer.clear();
224  }
225  frameScaleBuffer.shrink_to_fit();
226
227  [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:frameScaleBuffer.data()];
228
229  RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer =
230      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef];
231  XCTAssertEqual(scaledBuffer.width, outputSize.width);
232  XCTAssertEqual(scaledBuffer.height, outputSize.height);
233
234  if (outputSize.width > 0 && outputSize.height > 0) {
235    RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420];
236    RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420];
237    double psnr =
238        I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]);
239    XCTAssertEqual(psnr, webrtc::kPerfectPSNR);
240  }
241
242  CVBufferRelease(pixelBufferRef);
243}
244
245- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat {
246  [self cropAndScaleTestWithRGBPixelFormat:pixelFormat cropX:0 cropY:0];
247}
248
249- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat cropX:(int)cropX cropY:(int)cropY {
250  CVPixelBufferRef pixelBufferRef = NULL;
251  CVPixelBufferCreate(NULL, 720, 1280, pixelFormat, NULL, &pixelBufferRef);
252
253  DrawGradientInRGBPixelBuffer(pixelBufferRef);
254
255  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
256      initWithPixelBuffer:pixelBufferRef
257             adaptedWidth:CVPixelBufferGetWidth(pixelBufferRef)
258            adaptedHeight:CVPixelBufferGetHeight(pixelBufferRef)
259                cropWidth:CVPixelBufferGetWidth(pixelBufferRef) - cropX
260               cropHeight:CVPixelBufferGetHeight(pixelBufferRef) - cropY
261                    cropX:cropX
262                    cropY:cropY];
263
264  XCTAssertEqual(buffer.width, 720);
265  XCTAssertEqual(buffer.height, 1280);
266
267  CVPixelBufferRef outputPixelBufferRef = NULL;
268  CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &outputPixelBufferRef);
269  [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:NULL];
270
271  RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer =
272      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef];
273  XCTAssertEqual(scaledBuffer.width, 360);
274  XCTAssertEqual(scaledBuffer.height, 640);
275
276  RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420];
277  RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420];
278  double psnr =
279      I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]);
280  XCTAssertEqual(psnr, webrtc::kPerfectPSNR);
281
282  CVBufferRelease(pixelBufferRef);
283}
284
285- (void)toI420WithPixelFormat:(OSType)pixelFormat {
286  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(360, 640);
287
288  CVPixelBufferRef pixelBufferRef = NULL;
289  CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &pixelBufferRef);
290
291  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
292
293  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
294      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
295  RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [buffer toI420];
296
297  double psnr = I420PSNR(*i420Buffer, *[fromCVPixelBuffer nativeI420Buffer]);
298  double target = webrtc::kPerfectPSNR;
299  if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
300    // libyuv's I420ToRGB functions seem to lose some quality.
301    target = 19.0;
302  }
303  XCTAssertGreaterThanOrEqual(psnr, target);
304
305  CVBufferRelease(pixelBufferRef);
306}
307
308@end
309