1/*
2 *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCMTLVideoView.h"
12
13#import <Metal/Metal.h>
14#import <MetalKit/MetalKit.h>
15
16#import "base/RTCLogging.h"
17#import "base/RTCVideoFrame.h"
18#import "base/RTCVideoFrameBuffer.h"
19#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
20
21#import "RTCMTLI420Renderer.h"
22#import "RTCMTLNV12Renderer.h"
23#import "RTCMTLRGBRenderer.h"
24
25// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime.
26// Linking errors occur when compiling for architectures that don't support Metal.
27#define MTKViewClass NSClassFromString(@"MTKView")
28#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
29#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
30#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
31
32@interface RTC_OBJC_TYPE (RTCMTLVideoView)
33()<MTKViewDelegate> @property(nonatomic) RTCMTLI420Renderer *rendererI420;
34@property(nonatomic) RTCMTLNV12Renderer *rendererNV12;
35@property(nonatomic) RTCMTLRGBRenderer *rendererRGB;
36@property(nonatomic) MTKView *metalView;
37@property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame;
38@property(nonatomic) CGSize videoFrameSize;
39@property(nonatomic) int64_t lastFrameTimeNs;
40@end
41
42@implementation RTC_OBJC_TYPE (RTCMTLVideoView)
43
44@synthesize delegate = _delegate;
45@synthesize rendererI420 = _rendererI420;
46@synthesize rendererNV12 = _rendererNV12;
47@synthesize rendererRGB = _rendererRGB;
48@synthesize metalView = _metalView;
49@synthesize videoFrame = _videoFrame;
50@synthesize videoFrameSize = _videoFrameSize;
51@synthesize lastFrameTimeNs = _lastFrameTimeNs;
52@synthesize rotationOverride = _rotationOverride;
53
54- (instancetype)initWithFrame:(CGRect)frameRect {
55  self = [super initWithFrame:frameRect];
56  if (self) {
57    [self configure];
58  }
59  return self;
60}
61
62- (instancetype)initWithCoder:(NSCoder *)aCoder {
63  self = [super initWithCoder:aCoder];
64  if (self) {
65    [self configure];
66  }
67  return self;
68}
69
70- (BOOL)isEnabled {
71  return !self.metalView.paused;
72}
73
74- (void)setEnabled:(BOOL)enabled {
75  self.metalView.paused = !enabled;
76}
77
78- (UIViewContentMode)videoContentMode {
79  return self.metalView.contentMode;
80}
81
82- (void)setVideoContentMode:(UIViewContentMode)mode {
83  self.metalView.contentMode = mode;
84}
85
86#pragma mark - Private
87
88+ (BOOL)isMetalAvailable {
89#if defined(RTC_SUPPORTS_METAL)
90  return MTLCreateSystemDefaultDevice() != nil;
91#else
92  return NO;
93#endif
94}
95
96+ (MTKView *)createMetalView:(CGRect)frame {
97  return [[MTKViewClass alloc] initWithFrame:frame];
98}
99
100+ (RTCMTLNV12Renderer *)createNV12Renderer {
101  return [[RTCMTLNV12RendererClass alloc] init];
102}
103
104+ (RTCMTLI420Renderer *)createI420Renderer {
105  return [[RTCMTLI420RendererClass alloc] init];
106}
107
108+ (RTCMTLRGBRenderer *)createRGBRenderer {
109  return [[RTCMTLRGBRenderer alloc] init];
110}
111
112- (void)configure {
113  NSAssert([RTC_OBJC_TYPE(RTCMTLVideoView) isMetalAvailable],
114           @"Metal not availiable on this device");
115
116  self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds];
117  self.metalView.delegate = self;
118  self.metalView.contentMode = UIViewContentModeScaleAspectFill;
119  [self addSubview:self.metalView];
120  self.videoFrameSize = CGSizeZero;
121}
122
123- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled {
124    [super setMultipleTouchEnabled:multipleTouchEnabled];
125    self.metalView.multipleTouchEnabled = multipleTouchEnabled;
126}
127
128- (void)layoutSubviews {
129  [super layoutSubviews];
130
131  CGRect bounds = self.bounds;
132  self.metalView.frame = bounds;
133  if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) {
134    self.metalView.drawableSize = [self drawableSize];
135  } else {
136    self.metalView.drawableSize = bounds.size;
137  }
138}
139
140#pragma mark - MTKViewDelegate methods
141
142- (void)drawInMTKView:(nonnull MTKView *)view {
143  NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance.");
144  RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = self.videoFrame;
145  // Skip rendering if we've already rendered this frame.
146  if (!videoFrame || videoFrame.timeStampNs == self.lastFrameTimeNs) {
147    return;
148  }
149
150  if (CGRectIsEmpty(view.bounds)) {
151    return;
152  }
153
154  RTCMTLRenderer *renderer;
155  if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
156    RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer;
157    const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
158    if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
159      if (!self.rendererRGB) {
160        self.rendererRGB = [RTC_OBJC_TYPE(RTCMTLVideoView) createRGBRenderer];
161        if (![self.rendererRGB addRenderingDestination:self.metalView]) {
162          self.rendererRGB = nil;
163          RTCLogError(@"Failed to create RGB renderer");
164          return;
165        }
166      }
167      renderer = self.rendererRGB;
168    } else {
169      if (!self.rendererNV12) {
170        self.rendererNV12 = [RTC_OBJC_TYPE(RTCMTLVideoView) createNV12Renderer];
171        if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
172          self.rendererNV12 = nil;
173          RTCLogError(@"Failed to create NV12 renderer");
174          return;
175        }
176      }
177      renderer = self.rendererNV12;
178    }
179  } else {
180    if (!self.rendererI420) {
181      self.rendererI420 = [RTC_OBJC_TYPE(RTCMTLVideoView) createI420Renderer];
182      if (![self.rendererI420 addRenderingDestination:self.metalView]) {
183        self.rendererI420 = nil;
184        RTCLogError(@"Failed to create I420 renderer");
185        return;
186      }
187    }
188    renderer = self.rendererI420;
189  }
190
191  renderer.rotationOverride = self.rotationOverride;
192
193  [renderer drawFrame:videoFrame];
194  self.lastFrameTimeNs = videoFrame.timeStampNs;
195}
196
197- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
198}
199
200#pragma mark -
201
202- (void)setRotationOverride:(NSValue *)rotationOverride {
203  _rotationOverride = rotationOverride;
204
205  self.metalView.drawableSize = [self drawableSize];
206  [self setNeedsLayout];
207}
208
209- (RTCVideoRotation)frameRotation {
210  if (self.rotationOverride) {
211    RTCVideoRotation rotation;
212    if (@available(iOS 11, *)) {
213      [self.rotationOverride getValue:&rotation size:sizeof(rotation)];
214    } else {
215      [self.rotationOverride getValue:&rotation];
216    }
217    return rotation;
218  }
219
220  return self.videoFrame.rotation;
221}
222
223- (CGSize)drawableSize {
224  // Flip width/height if the rotations are not the same.
225  CGSize videoFrameSize = self.videoFrameSize;
226  RTCVideoRotation frameRotation = [self frameRotation];
227
228  BOOL useLandscape =
229      (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180);
230  BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) ||
231      (self.videoFrame.rotation == RTCVideoRotation_180);
232
233  if (useLandscape == sizeIsLandscape) {
234    return videoFrameSize;
235  } else {
236    return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
237  }
238}
239
240#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer)
241
242- (void)setSize:(CGSize)size {
243  __weak RTC_OBJC_TYPE(RTCMTLVideoView) *weakSelf = self;
244  dispatch_async(dispatch_get_main_queue(), ^{
245    RTC_OBJC_TYPE(RTCMTLVideoView) *strongSelf = weakSelf;
246
247    strongSelf.videoFrameSize = size;
248    CGSize drawableSize = [strongSelf drawableSize];
249
250    strongSelf.metalView.drawableSize = drawableSize;
251    [strongSelf setNeedsLayout];
252    [strongSelf.delegate videoView:self didChangeVideoSize:size];
253  });
254}
255
256- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
257  if (!self.isEnabled) {
258    return;
259  }
260
261  if (frame == nil) {
262    RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
263    return;
264  }
265  self.videoFrame = frame;
266}
267
268@end
269