1/*
2 *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCEAGLVideoView.h"
12
13#import <GLKit/GLKit.h>
14
15#import "RTCDefaultShader.h"
16#import "RTCDisplayLinkTimer.h"
17#import "RTCI420TextureCache.h"
18#import "RTCNV12TextureCache.h"
19#import "base/RTCLogging.h"
20#import "base/RTCVideoFrame.h"
21#import "base/RTCVideoFrameBuffer.h"
22#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
23
24// RTC_OBJC_TYPE(RTCEAGLVideoView) wraps a GLKView which is setup with
25// enableSetNeedsDisplay = NO for the purpose of gaining control of
26// exactly when to call -[GLKView display]. This need for extra
27// control is required to avoid triggering method calls on GLKView
28// that results in attempting to bind the underlying render buffer
29// when the drawable size would be empty which would result in the
30// error GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT. -[GLKView display] is
31// the method that will trigger the binding of the render
32// buffer. Because the standard behaviour of -[UIView setNeedsDisplay]
33// is disabled for the reasons above, the RTC_OBJC_TYPE(RTCEAGLVideoView) maintains
34// its own |isDirty| flag.
35
36@interface RTC_OBJC_TYPE (RTCEAGLVideoView)
37()<GLKViewDelegate>
38    // |videoFrame| is set when we receive a frame from a worker thread and is read
39    // from the display link callback so atomicity is required.
40    @property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame;
41@property(nonatomic, readonly) GLKView *glkView;
42@end
43
44@implementation RTC_OBJC_TYPE (RTCEAGLVideoView) {
45  RTCDisplayLinkTimer *_timer;
46  EAGLContext *_glContext;
47  // This flag should only be set and read on the main thread (e.g. by
48  // setNeedsDisplay)
49  BOOL _isDirty;
50  id<RTC_OBJC_TYPE(RTCVideoViewShading)> _shader;
51  RTCNV12TextureCache *_nv12TextureCache;
52  RTCI420TextureCache *_i420TextureCache;
53  // As timestamps should be unique between frames, will store last
54  // drawn frame timestamp instead of the whole frame to reduce memory usage.
55  int64_t _lastDrawnFrameTimeStampNs;
56}
57
58@synthesize delegate = _delegate;
59@synthesize videoFrame = _videoFrame;
60@synthesize glkView = _glkView;
61@synthesize rotationOverride = _rotationOverride;
62
63- (instancetype)initWithFrame:(CGRect)frame {
64  return [self initWithFrame:frame shader:[[RTCDefaultShader alloc] init]];
65}
66
67- (instancetype)initWithCoder:(NSCoder *)aDecoder {
68  return [self initWithCoder:aDecoder shader:[[RTCDefaultShader alloc] init]];
69}
70
71- (instancetype)initWithFrame:(CGRect)frame shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader {
72  if (self = [super initWithFrame:frame]) {
73    _shader = shader;
74    if (![self configure]) {
75      return nil;
76    }
77  }
78  return self;
79}
80
81- (instancetype)initWithCoder:(NSCoder *)aDecoder
82                       shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader {
83  if (self = [super initWithCoder:aDecoder]) {
84    _shader = shader;
85    if (![self configure]) {
86      return nil;
87    }
88  }
89  return self;
90}
91
92- (BOOL)configure {
93  EAGLContext *glContext =
94    [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3];
95  if (!glContext) {
96    glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
97  }
98  if (!glContext) {
99    RTCLogError(@"Failed to create EAGLContext");
100    return NO;
101  }
102  _glContext = glContext;
103
104  // GLKView manages a framebuffer for us.
105  _glkView = [[GLKView alloc] initWithFrame:CGRectZero
106                                    context:_glContext];
107  _glkView.drawableColorFormat = GLKViewDrawableColorFormatRGBA8888;
108  _glkView.drawableDepthFormat = GLKViewDrawableDepthFormatNone;
109  _glkView.drawableStencilFormat = GLKViewDrawableStencilFormatNone;
110  _glkView.drawableMultisample = GLKViewDrawableMultisampleNone;
111  _glkView.delegate = self;
112  _glkView.layer.masksToBounds = YES;
113  _glkView.enableSetNeedsDisplay = NO;
114  [self addSubview:_glkView];
115
116  // Listen to application state in order to clean up OpenGL before app goes
117  // away.
118  NSNotificationCenter *notificationCenter =
119    [NSNotificationCenter defaultCenter];
120  [notificationCenter addObserver:self
121                         selector:@selector(willResignActive)
122                             name:UIApplicationWillResignActiveNotification
123                           object:nil];
124  [notificationCenter addObserver:self
125                         selector:@selector(didBecomeActive)
126                             name:UIApplicationDidBecomeActiveNotification
127                           object:nil];
128
129  // Frames are received on a separate thread, so we poll for current frame
130  // using a refresh rate proportional to screen refresh frequency. This
131  // occurs on the main thread.
132  __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self;
133  _timer = [[RTCDisplayLinkTimer alloc] initWithTimerHandler:^{
134    RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf;
135    [strongSelf displayLinkTimerDidFire];
136  }];
137  if ([[UIApplication sharedApplication] applicationState] == UIApplicationStateActive) {
138    [self setupGL];
139  }
140  return YES;
141}
142
143- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled {
144    [super setMultipleTouchEnabled:multipleTouchEnabled];
145    _glkView.multipleTouchEnabled = multipleTouchEnabled;
146}
147
148- (void)dealloc {
149  [[NSNotificationCenter defaultCenter] removeObserver:self];
150  UIApplicationState appState =
151      [UIApplication sharedApplication].applicationState;
152  if (appState == UIApplicationStateActive) {
153    [self teardownGL];
154  }
155  [_timer invalidate];
156  [self ensureGLContext];
157  _shader = nil;
158  if (_glContext && [EAGLContext currentContext] == _glContext) {
159    [EAGLContext setCurrentContext:nil];
160  }
161}
162
163#pragma mark - UIView
164
165- (void)setNeedsDisplay {
166  [super setNeedsDisplay];
167  _isDirty = YES;
168}
169
170- (void)setNeedsDisplayInRect:(CGRect)rect {
171  [super setNeedsDisplayInRect:rect];
172  _isDirty = YES;
173}
174
175- (void)layoutSubviews {
176  [super layoutSubviews];
177  _glkView.frame = self.bounds;
178}
179
180#pragma mark - GLKViewDelegate
181
182// This method is called when the GLKView's content is dirty and needs to be
183// redrawn. This occurs on main thread.
184- (void)glkView:(GLKView *)view drawInRect:(CGRect)rect {
185  // The renderer will draw the frame to the framebuffer corresponding to the
186  // one used by |view|.
187  RTC_OBJC_TYPE(RTCVideoFrame) *frame = self.videoFrame;
188  if (!frame || frame.timeStampNs == _lastDrawnFrameTimeStampNs) {
189    return;
190  }
191  RTCVideoRotation rotation = frame.rotation;
192  if(_rotationOverride != nil) {
193    [_rotationOverride getValue: &rotation];
194  }
195  [self ensureGLContext];
196  glClear(GL_COLOR_BUFFER_BIT);
197  if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
198    if (!_nv12TextureCache) {
199      _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext];
200    }
201    if (_nv12TextureCache) {
202      [_nv12TextureCache uploadFrameToTextures:frame];
203      [_shader applyShadingForFrameWithWidth:frame.width
204                                      height:frame.height
205                                    rotation:rotation
206                                      yPlane:_nv12TextureCache.yTexture
207                                     uvPlane:_nv12TextureCache.uvTexture];
208      [_nv12TextureCache releaseTextures];
209
210      _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs;
211    }
212  } else {
213    if (!_i420TextureCache) {
214      _i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:_glContext];
215    }
216    [_i420TextureCache uploadFrameToTextures:frame];
217    [_shader applyShadingForFrameWithWidth:frame.width
218                                    height:frame.height
219                                  rotation:rotation
220                                    yPlane:_i420TextureCache.yTexture
221                                    uPlane:_i420TextureCache.uTexture
222                                    vPlane:_i420TextureCache.vTexture];
223
224    _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs;
225  }
226}
227
228#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer)
229
230// These methods may be called on non-main thread.
231- (void)setSize:(CGSize)size {
232  __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self;
233  dispatch_async(dispatch_get_main_queue(), ^{
234    RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf;
235    [strongSelf.delegate videoView:strongSelf didChangeVideoSize:size];
236  });
237}
238
239- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
240  self.videoFrame = frame;
241}
242
243#pragma mark - Private
244
245- (void)displayLinkTimerDidFire {
246  // Don't render unless video frame have changed or the view content
247  // has explicitly been marked dirty.
248  if (!_isDirty && _lastDrawnFrameTimeStampNs == self.videoFrame.timeStampNs) {
249    return;
250  }
251
252  // Always reset isDirty at this point, even if -[GLKView display]
253  // won't be called in the case the drawable size is empty.
254  _isDirty = NO;
255
256  // Only call -[GLKView display] if the drawable size is
257  // non-empty. Calling display will make the GLKView setup its
258  // render buffer if necessary, but that will fail with error
259  // GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT if size is empty.
260  if (self.bounds.size.width > 0 && self.bounds.size.height > 0) {
261    [_glkView display];
262  }
263}
264
265- (void)setupGL {
266  [self ensureGLContext];
267  glDisable(GL_DITHER);
268  _timer.isPaused = NO;
269}
270
271- (void)teardownGL {
272  self.videoFrame = nil;
273  _timer.isPaused = YES;
274  [_glkView deleteDrawable];
275  [self ensureGLContext];
276  _nv12TextureCache = nil;
277  _i420TextureCache = nil;
278}
279
280- (void)didBecomeActive {
281  [self setupGL];
282}
283
284- (void)willResignActive {
285  [self teardownGL];
286}
287
288- (void)ensureGLContext {
289  NSAssert(_glContext, @"context shouldn't be nil");
290  if ([EAGLContext currentContext] != _glContext) {
291    [EAGLContext setCurrentContext:_glContext];
292  }
293}
294
295@end
296