1/*
2 *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCOpenGLVideoRenderer.h"
12
13#include <string.h>
14
15#include "webrtc/base/scoped_ptr.h"
16
17#if TARGET_OS_IPHONE
18#import <OpenGLES/ES3/gl.h>
19#else
20#import <OpenGL/gl3.h>
21#endif
22
23#import "RTCVideoFrame.h"
24
25// TODO(tkchin): check and log openGL errors. Methods here return BOOLs in
26// anticipation of that happening in the future.
27
28#if TARGET_OS_IPHONE
29#define RTC_PIXEL_FORMAT GL_LUMINANCE
30#define SHADER_VERSION
31#define VERTEX_SHADER_IN "attribute"
32#define VERTEX_SHADER_OUT "varying"
33#define FRAGMENT_SHADER_IN "varying"
34#define FRAGMENT_SHADER_OUT
35#define FRAGMENT_SHADER_COLOR "gl_FragColor"
36#define FRAGMENT_SHADER_TEXTURE "texture2D"
37#else
38#define RTC_PIXEL_FORMAT GL_RED
39#define SHADER_VERSION "#version 150\n"
40#define VERTEX_SHADER_IN "in"
41#define VERTEX_SHADER_OUT "out"
42#define FRAGMENT_SHADER_IN "in"
43#define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n"
44#define FRAGMENT_SHADER_COLOR "fragColor"
45#define FRAGMENT_SHADER_TEXTURE "texture"
46#endif
47
48// Vertex shader doesn't do anything except pass coordinates through.
49static const char kVertexShaderSource[] =
50  SHADER_VERSION
51  VERTEX_SHADER_IN " vec2 position;\n"
52  VERTEX_SHADER_IN " vec2 texcoord;\n"
53  VERTEX_SHADER_OUT " vec2 v_texcoord;\n"
54  "void main() {\n"
55  "    gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n"
56  "    v_texcoord = texcoord;\n"
57  "}\n";
58
59// Fragment shader converts YUV values from input textures into a final RGB
60// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php.
61static const char kFragmentShaderSource[] =
62  SHADER_VERSION
63  "precision highp float;"
64  FRAGMENT_SHADER_IN " vec2 v_texcoord;\n"
65  "uniform lowp sampler2D s_textureY;\n"
66  "uniform lowp sampler2D s_textureU;\n"
67  "uniform lowp sampler2D s_textureV;\n"
68  FRAGMENT_SHADER_OUT
69  "void main() {\n"
70  "    float y, u, v, r, g, b;\n"
71  "    y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n"
72  "    u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n"
73  "    v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n"
74  "    u = u - 0.5;\n"
75  "    v = v - 0.5;\n"
76  "    r = y + 1.403 * v;\n"
77  "    g = y - 0.344 * u - 0.714 * v;\n"
78  "    b = y + 1.770 * u;\n"
79  "    " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n"
80  "  }\n";
81
82// Compiles a shader of the given |type| with GLSL source |source| and returns
83// the shader handle or 0 on error.
84GLuint CreateShader(GLenum type, const GLchar *source) {
85  GLuint shader = glCreateShader(type);
86  if (!shader) {
87    return 0;
88  }
89  glShaderSource(shader, 1, &source, NULL);
90  glCompileShader(shader);
91  GLint compileStatus = GL_FALSE;
92  glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus);
93  if (compileStatus == GL_FALSE) {
94    glDeleteShader(shader);
95    shader = 0;
96  }
97  return shader;
98}
99
100// Links a shader program with the given vertex and fragment shaders and
101// returns the program handle or 0 on error.
102GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) {
103  if (vertexShader == 0 || fragmentShader == 0) {
104    return 0;
105  }
106  GLuint program = glCreateProgram();
107  if (!program) {
108    return 0;
109  }
110  glAttachShader(program, vertexShader);
111  glAttachShader(program, fragmentShader);
112  glLinkProgram(program);
113  GLint linkStatus = GL_FALSE;
114  glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
115  if (linkStatus == GL_FALSE) {
116    glDeleteProgram(program);
117    program = 0;
118  }
119  return program;
120}
121
122// When modelview and projection matrices are identity (default) the world is
123// contained in the square around origin with unit size 2. Drawing to these
124// coordinates is equivalent to drawing to the entire screen. The texture is
125// stretched over that square using texture coordinates (u, v) that range
126// from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically
127// here because the incoming frame has origin in upper left hand corner but
128// OpenGL expects origin in bottom left corner.
129const GLfloat gVertices[] = {
130  // X, Y, U, V.
131  -1, -1, 0, 1,  // Bottom left.
132   1, -1, 1, 1,  // Bottom right.
133   1,  1, 1, 0,  // Top right.
134  -1,  1, 0, 0,  // Top left.
135};
136
137// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets
138// of 3 textures are used here, one for each of the Y, U and V planes. Having
139// two sets alleviates CPU blockage in the event that the GPU is asked to render
140// to a texture that is already in use.
141static const GLsizei kNumTextureSets = 2;
142static const GLsizei kNumTextures = 3 * kNumTextureSets;
143
144@implementation RTCOpenGLVideoRenderer {
145#if TARGET_OS_IPHONE
146  EAGLContext *_context;
147#else
148  NSOpenGLContext *_context;
149#endif
150  BOOL _isInitialized;
151  NSUInteger _currentTextureSet;
152  // Handles for OpenGL constructs.
153  GLuint _textures[kNumTextures];
154  GLuint _program;
155#if !TARGET_OS_IPHONE
156  GLuint _vertexArray;
157#endif
158  GLuint _vertexBuffer;
159  GLint _position;
160  GLint _texcoord;
161  GLint _ySampler;
162  GLint _uSampler;
163  GLint _vSampler;
164  // Used to create a non-padded plane for GPU upload when we receive padded
165  // frames.
166  rtc::scoped_ptr<uint8_t[]> _planeBuffer;
167}
168
169@synthesize lastDrawnFrame = _lastDrawnFrame;
170
171+ (void)initialize {
172  // Disable dithering for performance.
173  glDisable(GL_DITHER);
174}
175
176#if TARGET_OS_IPHONE
177- (instancetype)initWithContext:(EAGLContext *)context {
178#else
179- (instancetype)initWithContext:(NSOpenGLContext *)context {
180#endif
181  NSAssert(context != nil, @"context cannot be nil");
182  if (self = [super init]) {
183    _context = context;
184  }
185  return self;
186}
187
188- (BOOL)drawFrame:(RTCVideoFrame *)frame {
189  if (!_isInitialized) {
190    return NO;
191  }
192  if (_lastDrawnFrame == frame) {
193    return NO;
194  }
195  [self ensureGLContext];
196  glClear(GL_COLOR_BUFFER_BIT);
197  if (frame) {
198    if (![self updateTextureSizesForFrame:frame] ||
199        ![self updateTextureDataForFrame:frame]) {
200      return NO;
201    }
202#if !TARGET_OS_IPHONE
203    glBindVertexArray(_vertexArray);
204#endif
205    glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
206    glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
207  }
208#if !TARGET_OS_IPHONE
209  [_context flushBuffer];
210#endif
211  _lastDrawnFrame = frame;
212  return YES;
213}
214
215- (void)setupGL {
216  if (_isInitialized) {
217    return;
218  }
219  [self ensureGLContext];
220  if (![self setupProgram]) {
221    return;
222  }
223  if (![self setupTextures]) {
224    return;
225  }
226  if (![self setupVertices]) {
227    return;
228  }
229  glUseProgram(_program);
230  glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
231  _isInitialized = YES;
232}
233
234- (void)teardownGL {
235  if (!_isInitialized) {
236    return;
237  }
238  [self ensureGLContext];
239  glDeleteProgram(_program);
240  _program = 0;
241  glDeleteTextures(kNumTextures, _textures);
242  glDeleteBuffers(1, &_vertexBuffer);
243  _vertexBuffer = 0;
244#if !TARGET_OS_IPHONE
245  glDeleteVertexArrays(1, &_vertexArray);
246#endif
247  _isInitialized = NO;
248}
249
250#pragma mark - Private
251
252- (void)ensureGLContext {
253  NSAssert(_context, @"context shouldn't be nil");
254#if TARGET_OS_IPHONE
255  if ([EAGLContext currentContext] != _context) {
256    [EAGLContext setCurrentContext:_context];
257  }
258#else
259  if ([NSOpenGLContext currentContext] != _context) {
260    [_context makeCurrentContext];
261  }
262#endif
263}
264
265- (BOOL)setupProgram {
266  NSAssert(!_program, @"program already set up");
267  GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource);
268  NSAssert(vertexShader, @"failed to create vertex shader");
269  GLuint fragmentShader =
270      CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource);
271  NSAssert(fragmentShader, @"failed to create fragment shader");
272  _program = CreateProgram(vertexShader, fragmentShader);
273  // Shaders are created only to generate program.
274  if (vertexShader) {
275    glDeleteShader(vertexShader);
276  }
277  if (fragmentShader) {
278    glDeleteShader(fragmentShader);
279  }
280  if (!_program) {
281    return NO;
282  }
283  _position = glGetAttribLocation(_program, "position");
284  _texcoord = glGetAttribLocation(_program, "texcoord");
285  _ySampler = glGetUniformLocation(_program, "s_textureY");
286  _uSampler = glGetUniformLocation(_program, "s_textureU");
287  _vSampler = glGetUniformLocation(_program, "s_textureV");
288  if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 ||
289      _vSampler < 0) {
290    return NO;
291  }
292  return YES;
293}
294
295- (BOOL)setupTextures {
296  glGenTextures(kNumTextures, _textures);
297  // Set parameters for each of the textures we created.
298  for (GLsizei i = 0; i < kNumTextures; i++) {
299    glActiveTexture(GL_TEXTURE0 + i);
300    glBindTexture(GL_TEXTURE_2D, _textures[i]);
301    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
302    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
303    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
304    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
305  }
306  return YES;
307}
308
309- (BOOL)updateTextureSizesForFrame:(RTCVideoFrame *)frame {
310  if (frame.height == _lastDrawnFrame.height &&
311      frame.width == _lastDrawnFrame.width &&
312      frame.chromaWidth == _lastDrawnFrame.chromaWidth &&
313      frame.chromaHeight == _lastDrawnFrame.chromaHeight) {
314    return YES;
315  }
316  GLsizei lumaWidth = frame.width;
317  GLsizei lumaHeight = frame.height;
318  GLsizei chromaWidth = frame.chromaWidth;
319  GLsizei chromaHeight = frame.chromaHeight;
320  for (GLint i = 0; i < kNumTextureSets; i++) {
321    glActiveTexture(GL_TEXTURE0 + i * 3);
322    glTexImage2D(GL_TEXTURE_2D,
323                 0,
324                 RTC_PIXEL_FORMAT,
325                 lumaWidth,
326                 lumaHeight,
327                 0,
328                 RTC_PIXEL_FORMAT,
329                 GL_UNSIGNED_BYTE,
330                 0);
331    glActiveTexture(GL_TEXTURE0 + i * 3 + 1);
332    glTexImage2D(GL_TEXTURE_2D,
333                 0,
334                 RTC_PIXEL_FORMAT,
335                 chromaWidth,
336                 chromaHeight,
337                 0,
338                 RTC_PIXEL_FORMAT,
339                 GL_UNSIGNED_BYTE,
340                 0);
341    glActiveTexture(GL_TEXTURE0 + i * 3 + 2);
342    glTexImage2D(GL_TEXTURE_2D,
343                 0,
344                 RTC_PIXEL_FORMAT,
345                 chromaWidth,
346                 chromaHeight,
347                 0,
348                 RTC_PIXEL_FORMAT,
349                 GL_UNSIGNED_BYTE,
350                 0);
351  }
352  if ((NSUInteger)frame.yPitch != frame.width ||
353      (NSUInteger)frame.uPitch != frame.chromaWidth ||
354      (NSUInteger)frame.vPitch != frame.chromaWidth) {
355    _planeBuffer.reset(new uint8_t[frame.width * frame.height]);
356  } else {
357    _planeBuffer.reset();
358  }
359  return YES;
360}
361
362- (void)uploadPlane:(const uint8_t *)plane
363            sampler:(GLint)sampler
364             offset:(NSUInteger)offset
365              width:(size_t)width
366             height:(size_t)height
367             stride:(int32_t)stride {
368  glActiveTexture(GL_TEXTURE0 + offset);
369  // When setting texture sampler uniforms, the texture index is used not
370  // the texture handle.
371  glUniform1i(sampler, offset);
372#if TARGET_OS_IPHONE
373  BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3;
374#else
375  BOOL hasUnpackRowLength = YES;
376#endif
377  const uint8_t *uploadPlane = plane;
378  if ((size_t)stride != width) {
379   if (hasUnpackRowLength) {
380      // GLES3 allows us to specify stride.
381      glPixelStorei(GL_UNPACK_ROW_LENGTH, stride);
382      glTexImage2D(GL_TEXTURE_2D,
383                   0,
384                   RTC_PIXEL_FORMAT,
385                   width,
386                   height,
387                   0,
388                   RTC_PIXEL_FORMAT,
389                   GL_UNSIGNED_BYTE,
390                   uploadPlane);
391      glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
392      return;
393    } else {
394      // Make an unpadded copy and upload that instead. Quick profiling showed
395      // that this is faster than uploading row by row using glTexSubImage2D.
396      uint8_t *unpaddedPlane = _planeBuffer.get();
397      for (size_t y = 0; y < height; ++y) {
398        memcpy(unpaddedPlane + y * width, plane + y * stride, width);
399      }
400      uploadPlane = unpaddedPlane;
401    }
402  }
403  glTexImage2D(GL_TEXTURE_2D,
404               0,
405               RTC_PIXEL_FORMAT,
406               width,
407               height,
408               0,
409               RTC_PIXEL_FORMAT,
410               GL_UNSIGNED_BYTE,
411               uploadPlane);
412}
413
414- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame {
415  NSUInteger textureOffset = _currentTextureSet * 3;
416  NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset");
417
418  [self uploadPlane:frame.yPlane
419            sampler:_ySampler
420             offset:textureOffset
421              width:frame.width
422             height:frame.height
423             stride:frame.yPitch];
424
425  [self uploadPlane:frame.uPlane
426            sampler:_uSampler
427             offset:textureOffset + 1
428              width:frame.chromaWidth
429             height:frame.chromaHeight
430             stride:frame.uPitch];
431
432  [self uploadPlane:frame.vPlane
433            sampler:_vSampler
434             offset:textureOffset + 2
435              width:frame.chromaWidth
436             height:frame.chromaHeight
437             stride:frame.vPitch];
438
439  _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets;
440  return YES;
441}
442
443- (BOOL)setupVertices {
444#if !TARGET_OS_IPHONE
445  NSAssert(!_vertexArray, @"vertex array already set up");
446  glGenVertexArrays(1, &_vertexArray);
447  if (!_vertexArray) {
448    return NO;
449  }
450  glBindVertexArray(_vertexArray);
451#endif
452  NSAssert(!_vertexBuffer, @"vertex buffer already set up");
453  glGenBuffers(1, &_vertexBuffer);
454  if (!_vertexBuffer) {
455#if !TARGET_OS_IPHONE
456    glDeleteVertexArrays(1, &_vertexArray);
457    _vertexArray = 0;
458#endif
459    return NO;
460  }
461  glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
462  glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW);
463
464  // Read position attribute from |gVertices| with size of 2 and stride of 4
465  // beginning at the start of the array. The last argument indicates offset
466  // of data within |gVertices| as supplied to the vertex buffer.
467  glVertexAttribPointer(
468      _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0);
469  glEnableVertexAttribArray(_position);
470
471  // Read texcoord attribute from |gVertices| with size of 2 and stride of 4
472  // beginning at the first texcoord in the array. The last argument indicates
473  // offset of data within |gVertices| as supplied to the vertex buffer.
474  glVertexAttribPointer(_texcoord,
475                        2,
476                        GL_FLOAT,
477                        GL_FALSE,
478                        4 * sizeof(GLfloat),
479                        (void *)(2 * sizeof(GLfloat)));
480  glEnableVertexAttribArray(_texcoord);
481
482  return YES;
483}
484
485@end
486