1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "NativeWindowRenderer" 18#include "NativeWindowRenderer.h" 19 20#include <GLES2/gl2.h> 21#include <GLES2/gl2ext.h> 22#include <cutils/log.h> 23#include <gui/SurfaceTexture.h> 24#include <gui/SurfaceTextureClient.h> 25#include <media/stagefright/MediaBuffer.h> 26#include <media/stagefright/MetaData.h> 27#include <media/stagefright/foundation/ADebug.h> 28#include "VideoEditorTools.h" 29 30#define CHECK_EGL_ERROR CHECK(EGL_SUCCESS == eglGetError()) 31#define CHECK_GL_ERROR CHECK(GLenum(GL_NO_ERROR) == glGetError()) 32 33// 34// Vertex and fragment programs 35// 36 37// The matrix is derived from 38// frameworks/base/media/libstagefright/colorconversion/ColorConverter.cpp 39// 40// R * 255 = 1.164 * (Y - 16) + 1.596 * (V - 128) 41// G * 255 = 1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128) 42// B * 255 = 1.164 * (Y - 16) + 2.018 * (U - 128) 43// 44// Here we assume YUV are in the range of [0,255], RGB are in the range of 45// [0, 1] 46#define RGB2YUV_MATRIX \ 47"const mat4 rgb2yuv = mat4("\ 48" 65.52255, -37.79398, 111.98732, 0.00000,"\ 49" 128.62729, -74.19334, -93.81088, 0.00000,"\ 50" 24.92233, 111.98732, -18.17644, 0.00000,"\ 51" 16.00000, 128.00000, 128.00000, 1.00000);\n" 52 53#define YUV2RGB_MATRIX \ 54"const mat4 yuv2rgb = mat4("\ 55" 0.00456, 0.00456, 0.00456, 0.00000,"\ 56" 0.00000, -0.00153, 0.00791, 0.00000,"\ 57" 0.00626, -0.00319, 0.00000, 0.00000,"\ 58" -0.87416, 0.53133, -1.08599, 1.00000);\n" 59 60static const char vSrcNormal[] = 61 "attribute vec4 vPosition;\n" 62 "attribute vec2 vTexPos;\n" 63 "uniform mat4 texMatrix;\n" 64 "varying vec2 texCoords;\n" 65 "varying float topDown;\n" 66 "void main() {\n" 67 " gl_Position = vPosition;\n" 68 " texCoords = (texMatrix * vec4(vTexPos, 0.0, 1.0)).xy;\n" 69 " topDown = vTexPos.y;\n" 70 "}\n"; 71 72static const char fSrcNormal[] = 73 "#extension GL_OES_EGL_image_external : require\n" 74 "precision mediump float;\n" 75 "uniform samplerExternalOES texSampler;\n" 76 "varying vec2 texCoords;\n" 77 "void main() {\n" 78 " gl_FragColor = texture2D(texSampler, texCoords);\n" 79 "}\n"; 80 81static const char fSrcSepia[] = 82 "#extension GL_OES_EGL_image_external : require\n" 83 "precision mediump float;\n" 84 "uniform samplerExternalOES texSampler;\n" 85 "varying vec2 texCoords;\n" 86 RGB2YUV_MATRIX 87 YUV2RGB_MATRIX 88 "void main() {\n" 89 " vec4 rgb = texture2D(texSampler, texCoords);\n" 90 " vec4 yuv = rgb2yuv * rgb;\n" 91 " yuv = vec4(yuv.x, 117.0, 139.0, 1.0);\n" 92 " gl_FragColor = yuv2rgb * yuv;\n" 93 "}\n"; 94 95static const char fSrcNegative[] = 96 "#extension GL_OES_EGL_image_external : require\n" 97 "precision mediump float;\n" 98 "uniform samplerExternalOES texSampler;\n" 99 "varying vec2 texCoords;\n" 100 RGB2YUV_MATRIX 101 YUV2RGB_MATRIX 102 "void main() {\n" 103 " vec4 rgb = texture2D(texSampler, texCoords);\n" 104 " vec4 yuv = rgb2yuv * rgb;\n" 105 " yuv = vec4(255.0 - yuv.x, yuv.y, yuv.z, 1.0);\n" 106 " gl_FragColor = yuv2rgb * yuv;\n" 107 "}\n"; 108 109static const char fSrcGradient[] = 110 "#extension GL_OES_EGL_image_external : require\n" 111 "precision mediump float;\n" 112 "uniform samplerExternalOES texSampler;\n" 113 "varying vec2 texCoords;\n" 114 "varying float topDown;\n" 115 RGB2YUV_MATRIX 116 YUV2RGB_MATRIX 117 "void main() {\n" 118 " vec4 rgb = texture2D(texSampler, texCoords);\n" 119 " vec4 yuv = rgb2yuv * rgb;\n" 120 " vec4 mixin = vec4(15.0/31.0, 59.0/63.0, 31.0/31.0, 1.0);\n" 121 " vec4 yuv2 = rgb2yuv * vec4((mixin.xyz * topDown), 1);\n" 122 " yuv = vec4(yuv.x, yuv2.y, yuv2.z, 1);\n" 123 " gl_FragColor = yuv2rgb * yuv;\n" 124 "}\n"; 125 126namespace android { 127 128NativeWindowRenderer::NativeWindowRenderer(sp<ANativeWindow> nativeWindow, 129 int width, int height) 130 : mNativeWindow(nativeWindow) 131 , mDstWidth(width) 132 , mDstHeight(height) 133 , mLastVideoEffect(-1) 134 , mNextTextureId(100) 135 , mActiveInputs(0) 136 , mThreadCmd(CMD_IDLE) { 137 createThread(threadStart, this); 138} 139 140// The functions below run in the GL thread. 141// 142// All GL-related work is done in this thread, and other threads send 143// requests to this thread using a command code. We expect most of the 144// time there will only be one thread sending in requests, so we let 145// other threads wait until the request is finished by GL thread. 146 147int NativeWindowRenderer::threadStart(void* self) { 148 ALOGD("create thread"); 149 ((NativeWindowRenderer*)self)->glThread(); 150 return 0; 151} 152 153void NativeWindowRenderer::glThread() { 154 initializeEGL(); 155 createPrograms(); 156 157 Mutex::Autolock autoLock(mLock); 158 bool quit = false; 159 while (!quit) { 160 switch (mThreadCmd) { 161 case CMD_IDLE: 162 mCond.wait(mLock); 163 continue; 164 case CMD_RENDER_INPUT: 165 render(mThreadRenderInput); 166 break; 167 case CMD_RESERVE_TEXTURE: 168 glBindTexture(GL_TEXTURE_EXTERNAL_OES, mThreadTextureId); 169 CHECK_GL_ERROR; 170 break; 171 case CMD_DELETE_TEXTURE: 172 glDeleteTextures(1, &mThreadTextureId); 173 break; 174 case CMD_QUIT: 175 terminateEGL(); 176 quit = true; 177 break; 178 } 179 // Tell the requester that the command is finished. 180 mThreadCmd = CMD_IDLE; 181 mCond.broadcast(); 182 } 183 ALOGD("quit"); 184} 185 186void NativeWindowRenderer::initializeEGL() { 187 mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); 188 CHECK_EGL_ERROR; 189 190 EGLint majorVersion; 191 EGLint minorVersion; 192 eglInitialize(mEglDisplay, &majorVersion, &minorVersion); 193 CHECK_EGL_ERROR; 194 195 EGLConfig config; 196 EGLint numConfigs = -1; 197 EGLint configAttribs[] = { 198 EGL_SURFACE_TYPE, EGL_WINDOW_BIT, 199 EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, 200 EGL_RED_SIZE, 8, 201 EGL_GREEN_SIZE, 8, 202 EGL_BLUE_SIZE, 8, 203 EGL_NONE 204 }; 205 eglChooseConfig(mEglDisplay, configAttribs, &config, 1, &numConfigs); 206 CHECK_EGL_ERROR; 207 208 mEglSurface = eglCreateWindowSurface(mEglDisplay, config, 209 mNativeWindow.get(), NULL); 210 CHECK_EGL_ERROR; 211 212 EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE }; 213 mEglContext = eglCreateContext(mEglDisplay, config, EGL_NO_CONTEXT, 214 contextAttribs); 215 CHECK_EGL_ERROR; 216 217 eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext); 218 CHECK_EGL_ERROR; 219} 220 221void NativeWindowRenderer::terminateEGL() { 222 eglDestroyContext(mEglDisplay, mEglContext); 223 eglDestroySurface(mEglDisplay, mEglSurface); 224 eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); 225 eglTerminate(mEglDisplay); 226} 227 228void NativeWindowRenderer::createPrograms() { 229 GLuint vShader; 230 loadShader(GL_VERTEX_SHADER, vSrcNormal, &vShader); 231 232 const char* fSrc[NUMBER_OF_EFFECTS] = { 233 fSrcNormal, fSrcSepia, fSrcNegative, fSrcGradient 234 }; 235 236 for (int i = 0; i < NUMBER_OF_EFFECTS; i++) { 237 GLuint fShader; 238 loadShader(GL_FRAGMENT_SHADER, fSrc[i], &fShader); 239 createProgram(vShader, fShader, &mProgram[i]); 240 glDeleteShader(fShader); 241 CHECK_GL_ERROR; 242 } 243 244 glDeleteShader(vShader); 245 CHECK_GL_ERROR; 246} 247 248void NativeWindowRenderer::createProgram( 249 GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm) { 250 251 GLuint program = glCreateProgram(); 252 CHECK_GL_ERROR; 253 254 glAttachShader(program, vertexShader); 255 CHECK_GL_ERROR; 256 257 glAttachShader(program, fragmentShader); 258 CHECK_GL_ERROR; 259 260 glLinkProgram(program); 261 CHECK_GL_ERROR; 262 263 GLint linkStatus = GL_FALSE; 264 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); 265 if (linkStatus != GL_TRUE) { 266 GLint infoLen = 0; 267 glGetProgramiv(program, GL_INFO_LOG_LENGTH, &infoLen); 268 if (infoLen) { 269 char* buf = (char*) malloc(infoLen); 270 if (buf) { 271 glGetProgramInfoLog(program, infoLen, NULL, buf); 272 ALOGE("Program link log:\n%s\n", buf); 273 free(buf); 274 } 275 } 276 glDeleteProgram(program); 277 program = 0; 278 } 279 280 *outPgm = program; 281} 282 283void NativeWindowRenderer::loadShader(GLenum shaderType, const char* pSource, 284 GLuint* outShader) { 285 GLuint shader = glCreateShader(shaderType); 286 CHECK_GL_ERROR; 287 288 glShaderSource(shader, 1, &pSource, NULL); 289 CHECK_GL_ERROR; 290 291 glCompileShader(shader); 292 CHECK_GL_ERROR; 293 294 GLint compiled = 0; 295 glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); 296 if (!compiled) { 297 GLint infoLen = 0; 298 glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); 299 char* buf = (char*) malloc(infoLen); 300 if (buf) { 301 glGetShaderInfoLog(shader, infoLen, NULL, buf); 302 ALOGE("Shader compile log:\n%s\n", buf); 303 free(buf); 304 } 305 glDeleteShader(shader); 306 shader = 0; 307 } 308 *outShader = shader; 309} 310 311NativeWindowRenderer::~NativeWindowRenderer() { 312 CHECK(mActiveInputs == 0); 313 startRequest(CMD_QUIT); 314 sendRequest(); 315} 316 317void NativeWindowRenderer::render(RenderInput* input) { 318 sp<SurfaceTexture> ST = input->mST; 319 sp<SurfaceTextureClient> STC = input->mSTC; 320 321 if (input->mIsExternalBuffer) { 322 queueExternalBuffer(STC.get(), input->mBuffer, 323 input->mWidth, input->mHeight); 324 } else { 325 queueInternalBuffer(STC.get(), input->mBuffer); 326 } 327 328 ST->updateTexImage(); 329 glClearColor(0, 0, 0, 0); 330 glClear(GL_COLOR_BUFFER_BIT); 331 332 calculatePositionCoordinates(input->mRenderingMode, 333 input->mWidth, input->mHeight); 334 335 const GLfloat textureCoordinates[] = { 336 0.0f, 1.0f, 337 0.0f, 0.0f, 338 1.0f, 0.0f, 339 1.0f, 1.0f, 340 }; 341 342 updateProgramAndHandle(input->mVideoEffect); 343 344 glVertexAttribPointer(mPositionHandle, 2, GL_FLOAT, GL_FALSE, 0, 345 mPositionCoordinates); 346 CHECK_GL_ERROR; 347 348 glEnableVertexAttribArray(mPositionHandle); 349 CHECK_GL_ERROR; 350 351 glVertexAttribPointer(mTexPosHandle, 2, GL_FLOAT, GL_FALSE, 0, 352 textureCoordinates); 353 CHECK_GL_ERROR; 354 355 glEnableVertexAttribArray(mTexPosHandle); 356 CHECK_GL_ERROR; 357 358 GLfloat texMatrix[16]; 359 ST->getTransformMatrix(texMatrix); 360 glUniformMatrix4fv(mTexMatrixHandle, 1, GL_FALSE, texMatrix); 361 CHECK_GL_ERROR; 362 363 glBindTexture(GL_TEXTURE_EXTERNAL_OES, input->mTextureId); 364 CHECK_GL_ERROR; 365 366 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 367 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 368 glTexParameteri( 369 GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 370 glTexParameteri( 371 GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 372 CHECK_GL_ERROR; 373 374 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); 375 CHECK_GL_ERROR; 376 377 eglSwapBuffers(mEglDisplay, mEglSurface); 378} 379 380void NativeWindowRenderer::queueInternalBuffer(ANativeWindow *anw, 381 MediaBuffer* buffer) { 382 int64_t timeUs; 383 CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); 384 native_window_set_buffers_timestamp(anw, timeUs * 1000); 385 status_t err = anw->queueBuffer(anw, buffer->graphicBuffer().get(), -1); 386 if (err != 0) { 387 ALOGE("queueBuffer failed with error %s (%d)", strerror(-err), -err); 388 return; 389 } 390 391 sp<MetaData> metaData = buffer->meta_data(); 392 metaData->setInt32(kKeyRendered, 1); 393} 394 395void NativeWindowRenderer::queueExternalBuffer(ANativeWindow* anw, 396 MediaBuffer* buffer, int width, int height) { 397 native_window_set_buffers_geometry(anw, width, height, 398 HAL_PIXEL_FORMAT_YV12); 399 native_window_set_usage(anw, GRALLOC_USAGE_SW_WRITE_OFTEN); 400 401 ANativeWindowBuffer* anb; 402 CHECK(NO_ERROR == native_window_dequeue_buffer_and_wait(anw, &anb)); 403 CHECK(anb != NULL); 404 405 // Copy the buffer 406 uint8_t* img = NULL; 407 sp<GraphicBuffer> buf(new GraphicBuffer(anb, false)); 408 buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img)); 409 copyI420Buffer(buffer, img, width, height, buf->getStride()); 410 buf->unlock(); 411 CHECK(NO_ERROR == anw->queueBuffer(anw, buf->getNativeBuffer(), -1)); 412} 413 414void NativeWindowRenderer::copyI420Buffer(MediaBuffer* src, uint8_t* dst, 415 int srcWidth, int srcHeight, int stride) { 416 int strideUV = (stride / 2 + 0xf) & ~0xf; 417 uint8_t* p = (uint8_t*)src->data() + src->range_offset(); 418 // Y 419 for (int i = srcHeight; i > 0; i--) { 420 memcpy(dst, p, srcWidth); 421 dst += stride; 422 p += srcWidth; 423 } 424 // The src is I420, the dst is YV12. 425 // U 426 p += srcWidth * srcHeight / 4; 427 for (int i = srcHeight / 2; i > 0; i--) { 428 memcpy(dst, p, srcWidth / 2); 429 dst += strideUV; 430 p += srcWidth / 2; 431 } 432 // V 433 p -= srcWidth * srcHeight / 2; 434 for (int i = srcHeight / 2; i > 0; i--) { 435 memcpy(dst, p, srcWidth / 2); 436 dst += strideUV; 437 p += srcWidth / 2; 438 } 439} 440 441void NativeWindowRenderer::updateProgramAndHandle(uint32_t videoEffect) { 442 if (mLastVideoEffect == videoEffect) { 443 return; 444 } 445 446 mLastVideoEffect = videoEffect; 447 int i; 448 switch (mLastVideoEffect) { 449 case VIDEO_EFFECT_NONE: 450 i = 0; 451 break; 452 case VIDEO_EFFECT_SEPIA: 453 i = 1; 454 break; 455 case VIDEO_EFFECT_NEGATIVE: 456 i = 2; 457 break; 458 case VIDEO_EFFECT_GRADIENT: 459 i = 3; 460 break; 461 default: 462 i = 0; 463 break; 464 } 465 glUseProgram(mProgram[i]); 466 CHECK_GL_ERROR; 467 468 mPositionHandle = glGetAttribLocation(mProgram[i], "vPosition"); 469 mTexPosHandle = glGetAttribLocation(mProgram[i], "vTexPos"); 470 mTexMatrixHandle = glGetUniformLocation(mProgram[i], "texMatrix"); 471 CHECK_GL_ERROR; 472} 473 474void NativeWindowRenderer::calculatePositionCoordinates( 475 M4xVSS_MediaRendering renderingMode, int srcWidth, int srcHeight) { 476 float x, y; 477 switch (renderingMode) { 478 case M4xVSS_kResizing: 479 default: 480 x = 1; 481 y = 1; 482 break; 483 case M4xVSS_kCropping: 484 x = float(srcWidth) / mDstWidth; 485 y = float(srcHeight) / mDstHeight; 486 // Make the smaller side 1 487 if (x > y) { 488 x /= y; 489 y = 1; 490 } else { 491 y /= x; 492 x = 1; 493 } 494 break; 495 case M4xVSS_kBlackBorders: 496 x = float(srcWidth) / mDstWidth; 497 y = float(srcHeight) / mDstHeight; 498 // Make the larger side 1 499 if (x > y) { 500 y /= x; 501 x = 1; 502 } else { 503 x /= y; 504 y = 1; 505 } 506 break; 507 } 508 509 mPositionCoordinates[0] = -x; 510 mPositionCoordinates[1] = y; 511 mPositionCoordinates[2] = -x; 512 mPositionCoordinates[3] = -y; 513 mPositionCoordinates[4] = x; 514 mPositionCoordinates[5] = -y; 515 mPositionCoordinates[6] = x; 516 mPositionCoordinates[7] = y; 517} 518 519// 520// The functions below run in other threads. 521// 522 523void NativeWindowRenderer::startRequest(int cmd) { 524 mLock.lock(); 525 while (mThreadCmd != CMD_IDLE) { 526 mCond.wait(mLock); 527 } 528 mThreadCmd = cmd; 529} 530 531void NativeWindowRenderer::sendRequest() { 532 mCond.broadcast(); 533 while (mThreadCmd != CMD_IDLE) { 534 mCond.wait(mLock); 535 } 536 mLock.unlock(); 537} 538 539RenderInput* NativeWindowRenderer::createRenderInput() { 540 ALOGD("new render input %d", mNextTextureId); 541 RenderInput* input = new RenderInput(this, mNextTextureId); 542 543 startRequest(CMD_RESERVE_TEXTURE); 544 mThreadTextureId = mNextTextureId; 545 sendRequest(); 546 547 mNextTextureId++; 548 mActiveInputs++; 549 return input; 550} 551 552void NativeWindowRenderer::destroyRenderInput(RenderInput* input) { 553 ALOGD("destroy render input %d", input->mTextureId); 554 GLuint textureId = input->mTextureId; 555 delete input; 556 557 startRequest(CMD_DELETE_TEXTURE); 558 mThreadTextureId = textureId; 559 sendRequest(); 560 561 mActiveInputs--; 562} 563 564// 565// RenderInput 566// 567 568RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId) 569 : mRenderer(renderer) 570 , mTextureId(textureId) { 571 mST = new SurfaceTexture(mTextureId); 572 mSTC = new SurfaceTextureClient(mST); 573 native_window_connect(mSTC.get(), NATIVE_WINDOW_API_MEDIA); 574} 575 576RenderInput::~RenderInput() { 577} 578 579ANativeWindow* RenderInput::getTargetWindow() { 580 return mSTC.get(); 581} 582 583void RenderInput::updateVideoSize(sp<MetaData> meta) { 584 CHECK(meta->findInt32(kKeyWidth, &mWidth)); 585 CHECK(meta->findInt32(kKeyHeight, &mHeight)); 586 587 int left, top, right, bottom; 588 if (meta->findRect(kKeyCropRect, &left, &top, &right, &bottom)) { 589 mWidth = right - left + 1; 590 mHeight = bottom - top + 1; 591 } 592 593 // If rotation degrees is 90 or 270, swap width and height 594 // (mWidth and mHeight are the _rotated_ source rectangle). 595 int32_t rotationDegrees; 596 if (!meta->findInt32(kKeyRotation, &rotationDegrees)) { 597 rotationDegrees = 0; 598 } 599 600 if (rotationDegrees == 90 || rotationDegrees == 270) { 601 int tmp = mWidth; 602 mWidth = mHeight; 603 mHeight = tmp; 604 } 605} 606 607void RenderInput::render(MediaBuffer* buffer, uint32_t videoEffect, 608 M4xVSS_MediaRendering renderingMode, bool isExternalBuffer) { 609 mVideoEffect = videoEffect; 610 mRenderingMode = renderingMode; 611 mIsExternalBuffer = isExternalBuffer; 612 mBuffer = buffer; 613 614 mRenderer->startRequest(NativeWindowRenderer::CMD_RENDER_INPUT); 615 mRenderer->mThreadRenderInput = this; 616 mRenderer->sendRequest(); 617} 618 619} // namespace android 620