1/* 2 * libjingle 3 * Copyright 2014 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#if !defined(__has_feature) || !__has_feature(objc_arc) 29#error "This file requires ARC support." 30#endif 31 32#import "RTCOpenGLVideoRenderer.h" 33 34#include <string.h> 35 36#include "webrtc/base/scoped_ptr.h" 37 38#if TARGET_OS_IPHONE 39#import <OpenGLES/ES3/gl.h> 40#else 41#import <OpenGL/gl3.h> 42#endif 43 44#import "RTCI420Frame.h" 45 46// TODO(tkchin): check and log openGL errors. Methods here return BOOLs in 47// anticipation of that happening in the future. 48 49#if TARGET_OS_IPHONE 50#define RTC_PIXEL_FORMAT GL_LUMINANCE 51#define SHADER_VERSION 52#define VERTEX_SHADER_IN "attribute" 53#define VERTEX_SHADER_OUT "varying" 54#define FRAGMENT_SHADER_IN "varying" 55#define FRAGMENT_SHADER_OUT 56#define FRAGMENT_SHADER_COLOR "gl_FragColor" 57#define FRAGMENT_SHADER_TEXTURE "texture2D" 58#else 59#define RTC_PIXEL_FORMAT GL_RED 60#define SHADER_VERSION "#version 150\n" 61#define VERTEX_SHADER_IN "in" 62#define VERTEX_SHADER_OUT "out" 63#define FRAGMENT_SHADER_IN "in" 64#define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" 65#define FRAGMENT_SHADER_COLOR "fragColor" 66#define FRAGMENT_SHADER_TEXTURE "texture" 67#endif 68 69// Vertex shader doesn't do anything except pass coordinates through. 70static const char kVertexShaderSource[] = 71 SHADER_VERSION 72 VERTEX_SHADER_IN " vec2 position;\n" 73 VERTEX_SHADER_IN " vec2 texcoord;\n" 74 VERTEX_SHADER_OUT " vec2 v_texcoord;\n" 75 "void main() {\n" 76 " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" 77 " v_texcoord = texcoord;\n" 78 "}\n"; 79 80// Fragment shader converts YUV values from input textures into a final RGB 81// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. 82static const char kFragmentShaderSource[] = 83 SHADER_VERSION 84 "precision highp float;" 85 FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" 86 "uniform lowp sampler2D s_textureY;\n" 87 "uniform lowp sampler2D s_textureU;\n" 88 "uniform lowp sampler2D s_textureV;\n" 89 FRAGMENT_SHADER_OUT 90 "void main() {\n" 91 " float y, u, v, r, g, b;\n" 92 " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" 93 " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" 94 " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" 95 " u = u - 0.5;\n" 96 " v = v - 0.5;\n" 97 " r = y + 1.403 * v;\n" 98 " g = y - 0.344 * u - 0.714 * v;\n" 99 " b = y + 1.770 * u;\n" 100 " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" 101 " }\n"; 102 103// Compiles a shader of the given |type| with GLSL source |source| and returns 104// the shader handle or 0 on error. 105GLuint CreateShader(GLenum type, const GLchar* source) { 106 GLuint shader = glCreateShader(type); 107 if (!shader) { 108 return 0; 109 } 110 glShaderSource(shader, 1, &source, NULL); 111 glCompileShader(shader); 112 GLint compileStatus = GL_FALSE; 113 glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); 114 if (compileStatus == GL_FALSE) { 115 glDeleteShader(shader); 116 shader = 0; 117 } 118 return shader; 119} 120 121// Links a shader program with the given vertex and fragment shaders and 122// returns the program handle or 0 on error. 123GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { 124 if (vertexShader == 0 || fragmentShader == 0) { 125 return 0; 126 } 127 GLuint program = glCreateProgram(); 128 if (!program) { 129 return 0; 130 } 131 glAttachShader(program, vertexShader); 132 glAttachShader(program, fragmentShader); 133 glLinkProgram(program); 134 GLint linkStatus = GL_FALSE; 135 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); 136 if (linkStatus == GL_FALSE) { 137 glDeleteProgram(program); 138 program = 0; 139 } 140 return program; 141} 142 143// When modelview and projection matrices are identity (default) the world is 144// contained in the square around origin with unit size 2. Drawing to these 145// coordinates is equivalent to drawing to the entire screen. The texture is 146// stretched over that square using texture coordinates (u, v) that range 147// from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically 148// here because the incoming frame has origin in upper left hand corner but 149// OpenGL expects origin in bottom left corner. 150const GLfloat gVertices[] = { 151 // X, Y, U, V. 152 -1, -1, 0, 1, // Bottom left. 153 1, -1, 1, 1, // Bottom right. 154 1, 1, 1, 0, // Top right. 155 -1, 1, 0, 0, // Top left. 156}; 157 158// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets 159// of 3 textures are used here, one for each of the Y, U and V planes. Having 160// two sets alleviates CPU blockage in the event that the GPU is asked to render 161// to a texture that is already in use. 162static const GLsizei kNumTextureSets = 2; 163static const GLsizei kNumTextures = 3 * kNumTextureSets; 164 165@implementation RTCOpenGLVideoRenderer { 166#if TARGET_OS_IPHONE 167 EAGLContext* _context; 168#else 169 NSOpenGLContext* _context; 170#endif 171 BOOL _isInitialized; 172 NSUInteger _currentTextureSet; 173 // Handles for OpenGL constructs. 174 GLuint _textures[kNumTextures]; 175 GLuint _program; 176#if !TARGET_OS_IPHONE 177 GLuint _vertexArray; 178#endif 179 GLuint _vertexBuffer; 180 GLint _position; 181 GLint _texcoord; 182 GLint _ySampler; 183 GLint _uSampler; 184 GLint _vSampler; 185 // Used to create a non-padded plane for GPU upload when we receive padded 186 // frames. 187 rtc::scoped_ptr<uint8_t[]> _planeBuffer; 188} 189 190+ (void)initialize { 191 // Disable dithering for performance. 192 glDisable(GL_DITHER); 193} 194 195#if TARGET_OS_IPHONE 196- (instancetype)initWithContext:(EAGLContext*)context { 197#else 198- (instancetype)initWithContext:(NSOpenGLContext*)context { 199#endif 200 NSAssert(context != nil, @"context cannot be nil"); 201 if (self = [super init]) { 202 _context = context; 203 } 204 return self; 205} 206 207- (BOOL)drawFrame:(RTCI420Frame*)frame { 208 if (!_isInitialized) { 209 return NO; 210 } 211 if (_lastDrawnFrame == frame) { 212 return NO; 213 } 214 [self ensureGLContext]; 215 glClear(GL_COLOR_BUFFER_BIT); 216 if (frame) { 217 if (![self updateTextureSizesForFrame:frame] || 218 ![self updateTextureDataForFrame:frame]) { 219 return NO; 220 } 221#if !TARGET_OS_IPHONE 222 glBindVertexArray(_vertexArray); 223#endif 224 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 225 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); 226 } 227#if !TARGET_OS_IPHONE 228 [_context flushBuffer]; 229#endif 230 _lastDrawnFrame = frame; 231 return YES; 232} 233 234- (void)setupGL { 235 if (_isInitialized) { 236 return; 237 } 238 [self ensureGLContext]; 239 if (![self setupProgram]) { 240 return; 241 } 242 if (![self setupTextures]) { 243 return; 244 } 245 if (![self setupVertices]) { 246 return; 247 } 248 glUseProgram(_program); 249 glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 250 _isInitialized = YES; 251} 252 253- (void)teardownGL { 254 if (!_isInitialized) { 255 return; 256 } 257 [self ensureGLContext]; 258 glDeleteProgram(_program); 259 _program = 0; 260 glDeleteTextures(kNumTextures, _textures); 261 glDeleteBuffers(1, &_vertexBuffer); 262 _vertexBuffer = 0; 263#if !TARGET_OS_IPHONE 264 glDeleteVertexArrays(1, &_vertexArray); 265#endif 266 _isInitialized = NO; 267} 268 269#pragma mark - Private 270 271- (void)ensureGLContext { 272 NSAssert(_context, @"context shouldn't be nil"); 273#if TARGET_OS_IPHONE 274 if ([EAGLContext currentContext] != _context) { 275 [EAGLContext setCurrentContext:_context]; 276 } 277#else 278 if ([NSOpenGLContext currentContext] != _context) { 279 [_context makeCurrentContext]; 280 } 281#endif 282} 283 284- (BOOL)setupProgram { 285 NSAssert(!_program, @"program already set up"); 286 GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); 287 NSAssert(vertexShader, @"failed to create vertex shader"); 288 GLuint fragmentShader = 289 CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); 290 NSAssert(fragmentShader, @"failed to create fragment shader"); 291 _program = CreateProgram(vertexShader, fragmentShader); 292 // Shaders are created only to generate program. 293 if (vertexShader) { 294 glDeleteShader(vertexShader); 295 } 296 if (fragmentShader) { 297 glDeleteShader(fragmentShader); 298 } 299 if (!_program) { 300 return NO; 301 } 302 _position = glGetAttribLocation(_program, "position"); 303 _texcoord = glGetAttribLocation(_program, "texcoord"); 304 _ySampler = glGetUniformLocation(_program, "s_textureY"); 305 _uSampler = glGetUniformLocation(_program, "s_textureU"); 306 _vSampler = glGetUniformLocation(_program, "s_textureV"); 307 if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || 308 _vSampler < 0) { 309 return NO; 310 } 311 return YES; 312} 313 314- (BOOL)setupTextures { 315 glGenTextures(kNumTextures, _textures); 316 // Set parameters for each of the textures we created. 317 for (GLsizei i = 0; i < kNumTextures; i++) { 318 glActiveTexture(GL_TEXTURE0 + i); 319 glBindTexture(GL_TEXTURE_2D, _textures[i]); 320 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 321 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 322 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 323 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 324 } 325 return YES; 326} 327 328- (BOOL)updateTextureSizesForFrame:(RTCI420Frame*)frame { 329 if (frame.height == _lastDrawnFrame.height && 330 frame.width == _lastDrawnFrame.width && 331 frame.chromaWidth == _lastDrawnFrame.chromaWidth && 332 frame.chromaHeight == _lastDrawnFrame.chromaHeight) { 333 return YES; 334 } 335 GLsizei lumaWidth = frame.width; 336 GLsizei lumaHeight = frame.height; 337 GLsizei chromaWidth = frame.chromaWidth; 338 GLsizei chromaHeight = frame.chromaHeight; 339 for (GLint i = 0; i < kNumTextureSets; i++) { 340 glActiveTexture(GL_TEXTURE0 + i * 3); 341 glTexImage2D(GL_TEXTURE_2D, 342 0, 343 RTC_PIXEL_FORMAT, 344 lumaWidth, 345 lumaHeight, 346 0, 347 RTC_PIXEL_FORMAT, 348 GL_UNSIGNED_BYTE, 349 0); 350 glActiveTexture(GL_TEXTURE0 + i * 3 + 1); 351 glTexImage2D(GL_TEXTURE_2D, 352 0, 353 RTC_PIXEL_FORMAT, 354 chromaWidth, 355 chromaHeight, 356 0, 357 RTC_PIXEL_FORMAT, 358 GL_UNSIGNED_BYTE, 359 0); 360 glActiveTexture(GL_TEXTURE0 + i * 3 + 2); 361 glTexImage2D(GL_TEXTURE_2D, 362 0, 363 RTC_PIXEL_FORMAT, 364 chromaWidth, 365 chromaHeight, 366 0, 367 RTC_PIXEL_FORMAT, 368 GL_UNSIGNED_BYTE, 369 0); 370 } 371 if (frame.yPitch != frame.width || frame.uPitch != frame.chromaWidth || 372 frame.vPitch != frame.chromaWidth) { 373 _planeBuffer.reset(new uint8_t[frame.width * frame.height]); 374 } else { 375 _planeBuffer.reset(); 376 } 377 return YES; 378} 379 380- (void)uploadPlane:(const uint8_t*)plane 381 sampler:(GLint)sampler 382 offset:(NSUInteger)offset 383 width:(NSUInteger)width 384 height:(NSUInteger)height 385 stride:(NSInteger)stride { 386 glActiveTexture(GL_TEXTURE0 + offset); 387 // When setting texture sampler uniforms, the texture index is used not 388 // the texture handle. 389 glUniform1i(sampler, offset); 390#if TARGET_OS_IPHONE 391 BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3; 392#else 393 BOOL hasUnpackRowLength = YES; 394#endif 395 const uint8_t* uploadPlane = plane; 396 if (stride != width) { 397 if (hasUnpackRowLength) { 398 // GLES3 allows us to specify stride. 399 glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); 400 glTexImage2D(GL_TEXTURE_2D, 401 0, 402 RTC_PIXEL_FORMAT, 403 width, 404 height, 405 0, 406 RTC_PIXEL_FORMAT, 407 GL_UNSIGNED_BYTE, 408 uploadPlane); 409 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); 410 return; 411 } else { 412 // Make an unpadded copy and upload that instead. Quick profiling showed 413 // that this is faster than uploading row by row using glTexSubImage2D. 414 uint8_t* unpaddedPlane = _planeBuffer.get(); 415 for (NSUInteger y = 0; y < height; ++y) { 416 memcpy(unpaddedPlane + y * width, plane + y * stride, width); 417 } 418 uploadPlane = unpaddedPlane; 419 } 420 } 421 glTexImage2D(GL_TEXTURE_2D, 422 0, 423 RTC_PIXEL_FORMAT, 424 width, 425 height, 426 0, 427 RTC_PIXEL_FORMAT, 428 GL_UNSIGNED_BYTE, 429 uploadPlane); 430} 431 432- (BOOL)updateTextureDataForFrame:(RTCI420Frame*)frame { 433 NSUInteger textureOffset = _currentTextureSet * 3; 434 NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); 435 436 [self uploadPlane:frame.yPlane 437 sampler:_ySampler 438 offset:textureOffset 439 width:frame.width 440 height:frame.height 441 stride:frame.yPitch]; 442 443 [self uploadPlane:frame.uPlane 444 sampler:_uSampler 445 offset:textureOffset + 1 446 width:frame.chromaWidth 447 height:frame.chromaHeight 448 stride:frame.uPitch]; 449 450 [self uploadPlane:frame.vPlane 451 sampler:_vSampler 452 offset:textureOffset + 2 453 width:frame.chromaWidth 454 height:frame.chromaHeight 455 stride:frame.vPitch]; 456 457 _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; 458 return YES; 459} 460 461- (BOOL)setupVertices { 462#if !TARGET_OS_IPHONE 463 NSAssert(!_vertexArray, @"vertex array already set up"); 464 glGenVertexArrays(1, &_vertexArray); 465 if (!_vertexArray) { 466 return NO; 467 } 468 glBindVertexArray(_vertexArray); 469#endif 470 NSAssert(!_vertexBuffer, @"vertex buffer already set up"); 471 glGenBuffers(1, &_vertexBuffer); 472 if (!_vertexBuffer) { 473#if !TARGET_OS_IPHONE 474 glDeleteVertexArrays(1, &_vertexArray); 475 _vertexArray = 0; 476#endif 477 return NO; 478 } 479 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 480 glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); 481 482 // Read position attribute from |gVertices| with size of 2 and stride of 4 483 // beginning at the start of the array. The last argument indicates offset 484 // of data within |gVertices| as supplied to the vertex buffer. 485 glVertexAttribPointer( 486 _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void*)0); 487 glEnableVertexAttribArray(_position); 488 489 // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 490 // beginning at the first texcoord in the array. The last argument indicates 491 // offset of data within |gVertices| as supplied to the vertex buffer. 492 glVertexAttribPointer(_texcoord, 493 2, 494 GL_FLOAT, 495 GL_FALSE, 496 4 * sizeof(GLfloat), 497 (void*)(2 * sizeof(GLfloat))); 498 glEnableVertexAttribArray(_texcoord); 499 500 return YES; 501} 502 503@end 504