1/* 2 * Copyright 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import "RTCOpenGLVideoRenderer.h" 12 13#include <string.h> 14 15#include "webrtc/base/scoped_ptr.h" 16 17#if TARGET_OS_IPHONE 18#import <OpenGLES/ES3/gl.h> 19#else 20#import <OpenGL/gl3.h> 21#endif 22 23#import "RTCVideoFrame.h" 24 25// TODO(tkchin): check and log openGL errors. Methods here return BOOLs in 26// anticipation of that happening in the future. 27 28#if TARGET_OS_IPHONE 29#define RTC_PIXEL_FORMAT GL_LUMINANCE 30#define SHADER_VERSION 31#define VERTEX_SHADER_IN "attribute" 32#define VERTEX_SHADER_OUT "varying" 33#define FRAGMENT_SHADER_IN "varying" 34#define FRAGMENT_SHADER_OUT 35#define FRAGMENT_SHADER_COLOR "gl_FragColor" 36#define FRAGMENT_SHADER_TEXTURE "texture2D" 37#else 38#define RTC_PIXEL_FORMAT GL_RED 39#define SHADER_VERSION "#version 150\n" 40#define VERTEX_SHADER_IN "in" 41#define VERTEX_SHADER_OUT "out" 42#define FRAGMENT_SHADER_IN "in" 43#define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" 44#define FRAGMENT_SHADER_COLOR "fragColor" 45#define FRAGMENT_SHADER_TEXTURE "texture" 46#endif 47 48// Vertex shader doesn't do anything except pass coordinates through. 49static const char kVertexShaderSource[] = 50 SHADER_VERSION 51 VERTEX_SHADER_IN " vec2 position;\n" 52 VERTEX_SHADER_IN " vec2 texcoord;\n" 53 VERTEX_SHADER_OUT " vec2 v_texcoord;\n" 54 "void main() {\n" 55 " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" 56 " v_texcoord = texcoord;\n" 57 "}\n"; 58 59// Fragment shader converts YUV values from input textures into a final RGB 60// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. 61static const char kFragmentShaderSource[] = 62 SHADER_VERSION 63 "precision highp float;" 64 FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" 65 "uniform lowp sampler2D s_textureY;\n" 66 "uniform lowp sampler2D s_textureU;\n" 67 "uniform lowp sampler2D s_textureV;\n" 68 FRAGMENT_SHADER_OUT 69 "void main() {\n" 70 " float y, u, v, r, g, b;\n" 71 " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" 72 " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" 73 " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" 74 " u = u - 0.5;\n" 75 " v = v - 0.5;\n" 76 " r = y + 1.403 * v;\n" 77 " g = y - 0.344 * u - 0.714 * v;\n" 78 " b = y + 1.770 * u;\n" 79 " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" 80 " }\n"; 81 82// Compiles a shader of the given |type| with GLSL source |source| and returns 83// the shader handle or 0 on error. 84GLuint CreateShader(GLenum type, const GLchar *source) { 85 GLuint shader = glCreateShader(type); 86 if (!shader) { 87 return 0; 88 } 89 glShaderSource(shader, 1, &source, NULL); 90 glCompileShader(shader); 91 GLint compileStatus = GL_FALSE; 92 glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); 93 if (compileStatus == GL_FALSE) { 94 glDeleteShader(shader); 95 shader = 0; 96 } 97 return shader; 98} 99 100// Links a shader program with the given vertex and fragment shaders and 101// returns the program handle or 0 on error. 102GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { 103 if (vertexShader == 0 || fragmentShader == 0) { 104 return 0; 105 } 106 GLuint program = glCreateProgram(); 107 if (!program) { 108 return 0; 109 } 110 glAttachShader(program, vertexShader); 111 glAttachShader(program, fragmentShader); 112 glLinkProgram(program); 113 GLint linkStatus = GL_FALSE; 114 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); 115 if (linkStatus == GL_FALSE) { 116 glDeleteProgram(program); 117 program = 0; 118 } 119 return program; 120} 121 122// When modelview and projection matrices are identity (default) the world is 123// contained in the square around origin with unit size 2. Drawing to these 124// coordinates is equivalent to drawing to the entire screen. The texture is 125// stretched over that square using texture coordinates (u, v) that range 126// from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically 127// here because the incoming frame has origin in upper left hand corner but 128// OpenGL expects origin in bottom left corner. 129const GLfloat gVertices[] = { 130 // X, Y, U, V. 131 -1, -1, 0, 1, // Bottom left. 132 1, -1, 1, 1, // Bottom right. 133 1, 1, 1, 0, // Top right. 134 -1, 1, 0, 0, // Top left. 135}; 136 137// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets 138// of 3 textures are used here, one for each of the Y, U and V planes. Having 139// two sets alleviates CPU blockage in the event that the GPU is asked to render 140// to a texture that is already in use. 141static const GLsizei kNumTextureSets = 2; 142static const GLsizei kNumTextures = 3 * kNumTextureSets; 143 144@implementation RTCOpenGLVideoRenderer { 145#if TARGET_OS_IPHONE 146 EAGLContext *_context; 147#else 148 NSOpenGLContext *_context; 149#endif 150 BOOL _isInitialized; 151 NSUInteger _currentTextureSet; 152 // Handles for OpenGL constructs. 153 GLuint _textures[kNumTextures]; 154 GLuint _program; 155#if !TARGET_OS_IPHONE 156 GLuint _vertexArray; 157#endif 158 GLuint _vertexBuffer; 159 GLint _position; 160 GLint _texcoord; 161 GLint _ySampler; 162 GLint _uSampler; 163 GLint _vSampler; 164 // Used to create a non-padded plane for GPU upload when we receive padded 165 // frames. 166 rtc::scoped_ptr<uint8_t[]> _planeBuffer; 167} 168 169@synthesize lastDrawnFrame = _lastDrawnFrame; 170 171+ (void)initialize { 172 // Disable dithering for performance. 173 glDisable(GL_DITHER); 174} 175 176#if TARGET_OS_IPHONE 177- (instancetype)initWithContext:(EAGLContext *)context { 178#else 179- (instancetype)initWithContext:(NSOpenGLContext *)context { 180#endif 181 NSAssert(context != nil, @"context cannot be nil"); 182 if (self = [super init]) { 183 _context = context; 184 } 185 return self; 186} 187 188- (BOOL)drawFrame:(RTCVideoFrame *)frame { 189 if (!_isInitialized) { 190 return NO; 191 } 192 if (_lastDrawnFrame == frame) { 193 return NO; 194 } 195 [self ensureGLContext]; 196 glClear(GL_COLOR_BUFFER_BIT); 197 if (frame) { 198 if (![self updateTextureSizesForFrame:frame] || 199 ![self updateTextureDataForFrame:frame]) { 200 return NO; 201 } 202#if !TARGET_OS_IPHONE 203 glBindVertexArray(_vertexArray); 204#endif 205 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 206 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); 207 } 208#if !TARGET_OS_IPHONE 209 [_context flushBuffer]; 210#endif 211 _lastDrawnFrame = frame; 212 return YES; 213} 214 215- (void)setupGL { 216 if (_isInitialized) { 217 return; 218 } 219 [self ensureGLContext]; 220 if (![self setupProgram]) { 221 return; 222 } 223 if (![self setupTextures]) { 224 return; 225 } 226 if (![self setupVertices]) { 227 return; 228 } 229 glUseProgram(_program); 230 glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 231 _isInitialized = YES; 232} 233 234- (void)teardownGL { 235 if (!_isInitialized) { 236 return; 237 } 238 [self ensureGLContext]; 239 glDeleteProgram(_program); 240 _program = 0; 241 glDeleteTextures(kNumTextures, _textures); 242 glDeleteBuffers(1, &_vertexBuffer); 243 _vertexBuffer = 0; 244#if !TARGET_OS_IPHONE 245 glDeleteVertexArrays(1, &_vertexArray); 246#endif 247 _isInitialized = NO; 248} 249 250#pragma mark - Private 251 252- (void)ensureGLContext { 253 NSAssert(_context, @"context shouldn't be nil"); 254#if TARGET_OS_IPHONE 255 if ([EAGLContext currentContext] != _context) { 256 [EAGLContext setCurrentContext:_context]; 257 } 258#else 259 if ([NSOpenGLContext currentContext] != _context) { 260 [_context makeCurrentContext]; 261 } 262#endif 263} 264 265- (BOOL)setupProgram { 266 NSAssert(!_program, @"program already set up"); 267 GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); 268 NSAssert(vertexShader, @"failed to create vertex shader"); 269 GLuint fragmentShader = 270 CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); 271 NSAssert(fragmentShader, @"failed to create fragment shader"); 272 _program = CreateProgram(vertexShader, fragmentShader); 273 // Shaders are created only to generate program. 274 if (vertexShader) { 275 glDeleteShader(vertexShader); 276 } 277 if (fragmentShader) { 278 glDeleteShader(fragmentShader); 279 } 280 if (!_program) { 281 return NO; 282 } 283 _position = glGetAttribLocation(_program, "position"); 284 _texcoord = glGetAttribLocation(_program, "texcoord"); 285 _ySampler = glGetUniformLocation(_program, "s_textureY"); 286 _uSampler = glGetUniformLocation(_program, "s_textureU"); 287 _vSampler = glGetUniformLocation(_program, "s_textureV"); 288 if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || 289 _vSampler < 0) { 290 return NO; 291 } 292 return YES; 293} 294 295- (BOOL)setupTextures { 296 glGenTextures(kNumTextures, _textures); 297 // Set parameters for each of the textures we created. 298 for (GLsizei i = 0; i < kNumTextures; i++) { 299 glActiveTexture(GL_TEXTURE0 + i); 300 glBindTexture(GL_TEXTURE_2D, _textures[i]); 301 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 302 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 303 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 304 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 305 } 306 return YES; 307} 308 309- (BOOL)updateTextureSizesForFrame:(RTCVideoFrame *)frame { 310 if (frame.height == _lastDrawnFrame.height && 311 frame.width == _lastDrawnFrame.width && 312 frame.chromaWidth == _lastDrawnFrame.chromaWidth && 313 frame.chromaHeight == _lastDrawnFrame.chromaHeight) { 314 return YES; 315 } 316 GLsizei lumaWidth = frame.width; 317 GLsizei lumaHeight = frame.height; 318 GLsizei chromaWidth = frame.chromaWidth; 319 GLsizei chromaHeight = frame.chromaHeight; 320 for (GLint i = 0; i < kNumTextureSets; i++) { 321 glActiveTexture(GL_TEXTURE0 + i * 3); 322 glTexImage2D(GL_TEXTURE_2D, 323 0, 324 RTC_PIXEL_FORMAT, 325 lumaWidth, 326 lumaHeight, 327 0, 328 RTC_PIXEL_FORMAT, 329 GL_UNSIGNED_BYTE, 330 0); 331 glActiveTexture(GL_TEXTURE0 + i * 3 + 1); 332 glTexImage2D(GL_TEXTURE_2D, 333 0, 334 RTC_PIXEL_FORMAT, 335 chromaWidth, 336 chromaHeight, 337 0, 338 RTC_PIXEL_FORMAT, 339 GL_UNSIGNED_BYTE, 340 0); 341 glActiveTexture(GL_TEXTURE0 + i * 3 + 2); 342 glTexImage2D(GL_TEXTURE_2D, 343 0, 344 RTC_PIXEL_FORMAT, 345 chromaWidth, 346 chromaHeight, 347 0, 348 RTC_PIXEL_FORMAT, 349 GL_UNSIGNED_BYTE, 350 0); 351 } 352 if ((NSUInteger)frame.yPitch != frame.width || 353 (NSUInteger)frame.uPitch != frame.chromaWidth || 354 (NSUInteger)frame.vPitch != frame.chromaWidth) { 355 _planeBuffer.reset(new uint8_t[frame.width * frame.height]); 356 } else { 357 _planeBuffer.reset(); 358 } 359 return YES; 360} 361 362- (void)uploadPlane:(const uint8_t *)plane 363 sampler:(GLint)sampler 364 offset:(NSUInteger)offset 365 width:(size_t)width 366 height:(size_t)height 367 stride:(int32_t)stride { 368 glActiveTexture(GL_TEXTURE0 + offset); 369 // When setting texture sampler uniforms, the texture index is used not 370 // the texture handle. 371 glUniform1i(sampler, offset); 372#if TARGET_OS_IPHONE 373 BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3; 374#else 375 BOOL hasUnpackRowLength = YES; 376#endif 377 const uint8_t *uploadPlane = plane; 378 if ((size_t)stride != width) { 379 if (hasUnpackRowLength) { 380 // GLES3 allows us to specify stride. 381 glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); 382 glTexImage2D(GL_TEXTURE_2D, 383 0, 384 RTC_PIXEL_FORMAT, 385 width, 386 height, 387 0, 388 RTC_PIXEL_FORMAT, 389 GL_UNSIGNED_BYTE, 390 uploadPlane); 391 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); 392 return; 393 } else { 394 // Make an unpadded copy and upload that instead. Quick profiling showed 395 // that this is faster than uploading row by row using glTexSubImage2D. 396 uint8_t *unpaddedPlane = _planeBuffer.get(); 397 for (size_t y = 0; y < height; ++y) { 398 memcpy(unpaddedPlane + y * width, plane + y * stride, width); 399 } 400 uploadPlane = unpaddedPlane; 401 } 402 } 403 glTexImage2D(GL_TEXTURE_2D, 404 0, 405 RTC_PIXEL_FORMAT, 406 width, 407 height, 408 0, 409 RTC_PIXEL_FORMAT, 410 GL_UNSIGNED_BYTE, 411 uploadPlane); 412} 413 414- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame { 415 NSUInteger textureOffset = _currentTextureSet * 3; 416 NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); 417 418 [self uploadPlane:frame.yPlane 419 sampler:_ySampler 420 offset:textureOffset 421 width:frame.width 422 height:frame.height 423 stride:frame.yPitch]; 424 425 [self uploadPlane:frame.uPlane 426 sampler:_uSampler 427 offset:textureOffset + 1 428 width:frame.chromaWidth 429 height:frame.chromaHeight 430 stride:frame.uPitch]; 431 432 [self uploadPlane:frame.vPlane 433 sampler:_vSampler 434 offset:textureOffset + 2 435 width:frame.chromaWidth 436 height:frame.chromaHeight 437 stride:frame.vPitch]; 438 439 _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; 440 return YES; 441} 442 443- (BOOL)setupVertices { 444#if !TARGET_OS_IPHONE 445 NSAssert(!_vertexArray, @"vertex array already set up"); 446 glGenVertexArrays(1, &_vertexArray); 447 if (!_vertexArray) { 448 return NO; 449 } 450 glBindVertexArray(_vertexArray); 451#endif 452 NSAssert(!_vertexBuffer, @"vertex buffer already set up"); 453 glGenBuffers(1, &_vertexBuffer); 454 if (!_vertexBuffer) { 455#if !TARGET_OS_IPHONE 456 glDeleteVertexArrays(1, &_vertexArray); 457 _vertexArray = 0; 458#endif 459 return NO; 460 } 461 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 462 glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); 463 464 // Read position attribute from |gVertices| with size of 2 and stride of 4 465 // beginning at the start of the array. The last argument indicates offset 466 // of data within |gVertices| as supplied to the vertex buffer. 467 glVertexAttribPointer( 468 _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); 469 glEnableVertexAttribArray(_position); 470 471 // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 472 // beginning at the first texcoord in the array. The last argument indicates 473 // offset of data within |gVertices| as supplied to the vertex buffer. 474 glVertexAttribPointer(_texcoord, 475 2, 476 GL_FLOAT, 477 GL_FALSE, 478 4 * sizeof(GLfloat), 479 (void *)(2 * sizeof(GLfloat))); 480 glEnableVertexAttribArray(_texcoord); 481 482 return YES; 483} 484 485@end 486