1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "NativeWindowRenderer"
18#include "NativeWindowRenderer.h"
19
20#include <GLES2/gl2.h>
21#include <GLES2/gl2ext.h>
22#include <cutils/log.h>
23#include <gui/SurfaceTexture.h>
24#include <gui/SurfaceTextureClient.h>
25#include <media/stagefright/MediaBuffer.h>
26#include <media/stagefright/MetaData.h>
27#include <media/stagefright/foundation/ADebug.h>
28#include "VideoEditorTools.h"
29
30#define CHECK_EGL_ERROR CHECK(EGL_SUCCESS == eglGetError())
31#define CHECK_GL_ERROR CHECK(GLenum(GL_NO_ERROR) == glGetError())
32
33//
34// Vertex and fragment programs
35//
36
37// The matrix is derived from
38// frameworks/base/media/libstagefright/colorconversion/ColorConverter.cpp
39//
40// R * 255 = 1.164 * (Y - 16) + 1.596 * (V - 128)
41// G * 255 = 1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128)
42// B * 255 = 1.164 * (Y - 16) + 2.018 * (U - 128)
43//
44// Here we assume YUV are in the range of [0,255], RGB are in the range of
45// [0, 1]
46#define RGB2YUV_MATRIX \
47"const mat4 rgb2yuv = mat4("\
48"    65.52255,   -37.79398,   111.98732,     0.00000,"\
49"   128.62729,   -74.19334,   -93.81088,     0.00000,"\
50"    24.92233,   111.98732,   -18.17644,     0.00000,"\
51"    16.00000,   128.00000,   128.00000,     1.00000);\n"
52
53#define YUV2RGB_MATRIX \
54"const mat4 yuv2rgb = mat4("\
55"   0.00456,   0.00456,   0.00456,   0.00000,"\
56"   0.00000,  -0.00153,   0.00791,   0.00000,"\
57"   0.00626,  -0.00319,   0.00000,   0.00000,"\
58"  -0.87416,   0.53133,  -1.08599,   1.00000);\n"
59
60static const char vSrcNormal[] =
61    "attribute vec4 vPosition;\n"
62    "attribute vec2 vTexPos;\n"
63    "uniform mat4 texMatrix;\n"
64    "varying vec2 texCoords;\n"
65    "varying float topDown;\n"
66    "void main() {\n"
67    "  gl_Position = vPosition;\n"
68    "  texCoords = (texMatrix * vec4(vTexPos, 0.0, 1.0)).xy;\n"
69    "  topDown = vTexPos.y;\n"
70    "}\n";
71
72static const char fSrcNormal[] =
73    "#extension GL_OES_EGL_image_external : require\n"
74    "precision mediump float;\n"
75    "uniform samplerExternalOES texSampler;\n"
76    "varying vec2 texCoords;\n"
77    "void main() {\n"
78    "  gl_FragColor = texture2D(texSampler, texCoords);\n"
79    "}\n";
80
81static const char fSrcSepia[] =
82    "#extension GL_OES_EGL_image_external : require\n"
83    "precision mediump float;\n"
84    "uniform samplerExternalOES texSampler;\n"
85    "varying vec2 texCoords;\n"
86    RGB2YUV_MATRIX
87    YUV2RGB_MATRIX
88    "void main() {\n"
89    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
90    "  vec4 yuv = rgb2yuv * rgb;\n"
91    "  yuv = vec4(yuv.x, 117.0, 139.0, 1.0);\n"
92    "  gl_FragColor = yuv2rgb * yuv;\n"
93    "}\n";
94
95static const char fSrcNegative[] =
96    "#extension GL_OES_EGL_image_external : require\n"
97    "precision mediump float;\n"
98    "uniform samplerExternalOES texSampler;\n"
99    "varying vec2 texCoords;\n"
100    RGB2YUV_MATRIX
101    YUV2RGB_MATRIX
102    "void main() {\n"
103    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
104    "  vec4 yuv = rgb2yuv * rgb;\n"
105    "  yuv = vec4(255.0 - yuv.x, yuv.y, yuv.z, 1.0);\n"
106    "  gl_FragColor = yuv2rgb * yuv;\n"
107    "}\n";
108
109static const char fSrcGradient[] =
110    "#extension GL_OES_EGL_image_external : require\n"
111    "precision mediump float;\n"
112    "uniform samplerExternalOES texSampler;\n"
113    "varying vec2 texCoords;\n"
114    "varying float topDown;\n"
115    RGB2YUV_MATRIX
116    YUV2RGB_MATRIX
117    "void main() {\n"
118    "  vec4 rgb = texture2D(texSampler, texCoords);\n"
119    "  vec4 yuv = rgb2yuv * rgb;\n"
120    "  vec4 mixin = vec4(15.0/31.0, 59.0/63.0, 31.0/31.0, 1.0);\n"
121    "  vec4 yuv2 = rgb2yuv * vec4((mixin.xyz * topDown), 1);\n"
122    "  yuv = vec4(yuv.x, yuv2.y, yuv2.z, 1);\n"
123    "  gl_FragColor = yuv2rgb * yuv;\n"
124    "}\n";
125
126namespace android {
127
128NativeWindowRenderer::NativeWindowRenderer(sp<ANativeWindow> nativeWindow,
129        int width, int height)
130    : mNativeWindow(nativeWindow)
131    , mDstWidth(width)
132    , mDstHeight(height)
133    , mLastVideoEffect(-1)
134    , mNextTextureId(100)
135    , mActiveInputs(0)
136    , mThreadCmd(CMD_IDLE) {
137    createThread(threadStart, this);
138}
139
140// The functions below run in the GL thread.
141//
142// All GL-related work is done in this thread, and other threads send
143// requests to this thread using a command code. We expect most of the
144// time there will only be one thread sending in requests, so we let
145// other threads wait until the request is finished by GL thread.
146
147int NativeWindowRenderer::threadStart(void* self) {
148    ALOGD("create thread");
149    ((NativeWindowRenderer*)self)->glThread();
150    return 0;
151}
152
153void NativeWindowRenderer::glThread() {
154    initializeEGL();
155    createPrograms();
156
157    Mutex::Autolock autoLock(mLock);
158    bool quit = false;
159    while (!quit) {
160        switch (mThreadCmd) {
161            case CMD_IDLE:
162                mCond.wait(mLock);
163                continue;
164            case CMD_RENDER_INPUT:
165                render(mThreadRenderInput);
166                break;
167            case CMD_RESERVE_TEXTURE:
168                glBindTexture(GL_TEXTURE_EXTERNAL_OES, mThreadTextureId);
169                CHECK_GL_ERROR;
170                break;
171            case CMD_DELETE_TEXTURE:
172                glDeleteTextures(1, &mThreadTextureId);
173                break;
174            case CMD_QUIT:
175                terminateEGL();
176                quit = true;
177                break;
178        }
179        // Tell the requester that the command is finished.
180        mThreadCmd = CMD_IDLE;
181        mCond.broadcast();
182    }
183    ALOGD("quit");
184}
185
186void NativeWindowRenderer::initializeEGL() {
187    mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
188    CHECK_EGL_ERROR;
189
190    EGLint majorVersion;
191    EGLint minorVersion;
192    eglInitialize(mEglDisplay, &majorVersion, &minorVersion);
193    CHECK_EGL_ERROR;
194
195    EGLConfig config;
196    EGLint numConfigs = -1;
197    EGLint configAttribs[] = {
198        EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
199        EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
200        EGL_RED_SIZE, 8,
201        EGL_GREEN_SIZE, 8,
202        EGL_BLUE_SIZE, 8,
203        EGL_NONE
204    };
205    eglChooseConfig(mEglDisplay, configAttribs, &config, 1, &numConfigs);
206    CHECK_EGL_ERROR;
207
208    mEglSurface = eglCreateWindowSurface(mEglDisplay, config,
209        mNativeWindow.get(), NULL);
210    CHECK_EGL_ERROR;
211
212    EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
213    mEglContext = eglCreateContext(mEglDisplay, config, EGL_NO_CONTEXT,
214        contextAttribs);
215    CHECK_EGL_ERROR;
216
217    eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext);
218    CHECK_EGL_ERROR;
219}
220
221void NativeWindowRenderer::terminateEGL() {
222    eglDestroyContext(mEglDisplay, mEglContext);
223    eglDestroySurface(mEglDisplay, mEglSurface);
224    eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
225    eglTerminate(mEglDisplay);
226}
227
228void NativeWindowRenderer::createPrograms() {
229    GLuint vShader;
230    loadShader(GL_VERTEX_SHADER, vSrcNormal, &vShader);
231
232    const char* fSrc[NUMBER_OF_EFFECTS] = {
233        fSrcNormal, fSrcSepia, fSrcNegative, fSrcGradient
234    };
235
236    for (int i = 0; i < NUMBER_OF_EFFECTS; i++) {
237        GLuint fShader;
238        loadShader(GL_FRAGMENT_SHADER, fSrc[i], &fShader);
239        createProgram(vShader, fShader, &mProgram[i]);
240        glDeleteShader(fShader);
241        CHECK_GL_ERROR;
242    }
243
244    glDeleteShader(vShader);
245    CHECK_GL_ERROR;
246}
247
248void NativeWindowRenderer::createProgram(
249    GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm) {
250
251    GLuint program = glCreateProgram();
252    CHECK_GL_ERROR;
253
254    glAttachShader(program, vertexShader);
255    CHECK_GL_ERROR;
256
257    glAttachShader(program, fragmentShader);
258    CHECK_GL_ERROR;
259
260    glLinkProgram(program);
261    CHECK_GL_ERROR;
262
263    GLint linkStatus = GL_FALSE;
264    glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
265    if (linkStatus != GL_TRUE) {
266        GLint infoLen = 0;
267        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &infoLen);
268        if (infoLen) {
269            char* buf = (char*) malloc(infoLen);
270            if (buf) {
271                glGetProgramInfoLog(program, infoLen, NULL, buf);
272                ALOGE("Program link log:\n%s\n", buf);
273                free(buf);
274            }
275        }
276        glDeleteProgram(program);
277        program = 0;
278    }
279
280    *outPgm = program;
281}
282
283void NativeWindowRenderer::loadShader(GLenum shaderType, const char* pSource,
284        GLuint* outShader) {
285    GLuint shader = glCreateShader(shaderType);
286    CHECK_GL_ERROR;
287
288    glShaderSource(shader, 1, &pSource, NULL);
289    CHECK_GL_ERROR;
290
291    glCompileShader(shader);
292    CHECK_GL_ERROR;
293
294    GLint compiled = 0;
295    glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
296    if (!compiled) {
297        GLint infoLen = 0;
298        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
299        char* buf = (char*) malloc(infoLen);
300        if (buf) {
301            glGetShaderInfoLog(shader, infoLen, NULL, buf);
302            ALOGE("Shader compile log:\n%s\n", buf);
303            free(buf);
304        }
305        glDeleteShader(shader);
306        shader = 0;
307    }
308    *outShader = shader;
309}
310
311NativeWindowRenderer::~NativeWindowRenderer() {
312    CHECK(mActiveInputs == 0);
313    startRequest(CMD_QUIT);
314    sendRequest();
315}
316
317void NativeWindowRenderer::render(RenderInput* input) {
318    sp<SurfaceTexture> ST = input->mST;
319    sp<SurfaceTextureClient> STC = input->mSTC;
320
321    if (input->mIsExternalBuffer) {
322        queueExternalBuffer(STC.get(), input->mBuffer,
323            input->mWidth, input->mHeight);
324    } else {
325        queueInternalBuffer(STC.get(), input->mBuffer);
326    }
327
328    ST->updateTexImage();
329    glClearColor(0, 0, 0, 0);
330    glClear(GL_COLOR_BUFFER_BIT);
331
332    calculatePositionCoordinates(input->mRenderingMode,
333        input->mWidth, input->mHeight);
334
335    const GLfloat textureCoordinates[] = {
336         0.0f,  1.0f,
337         0.0f,  0.0f,
338         1.0f,  0.0f,
339         1.0f,  1.0f,
340    };
341
342    updateProgramAndHandle(input->mVideoEffect);
343
344    glVertexAttribPointer(mPositionHandle, 2, GL_FLOAT, GL_FALSE, 0,
345        mPositionCoordinates);
346    CHECK_GL_ERROR;
347
348    glEnableVertexAttribArray(mPositionHandle);
349    CHECK_GL_ERROR;
350
351    glVertexAttribPointer(mTexPosHandle, 2, GL_FLOAT, GL_FALSE, 0,
352        textureCoordinates);
353    CHECK_GL_ERROR;
354
355    glEnableVertexAttribArray(mTexPosHandle);
356    CHECK_GL_ERROR;
357
358    GLfloat texMatrix[16];
359    ST->getTransformMatrix(texMatrix);
360    glUniformMatrix4fv(mTexMatrixHandle, 1, GL_FALSE, texMatrix);
361    CHECK_GL_ERROR;
362
363    glBindTexture(GL_TEXTURE_EXTERNAL_OES, input->mTextureId);
364    CHECK_GL_ERROR;
365
366    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
367    glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
368    glTexParameteri(
369        GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
370    glTexParameteri(
371        GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
372    CHECK_GL_ERROR;
373
374    glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
375    CHECK_GL_ERROR;
376
377    eglSwapBuffers(mEglDisplay, mEglSurface);
378}
379
380void NativeWindowRenderer::queueInternalBuffer(ANativeWindow *anw,
381    MediaBuffer* buffer) {
382    int64_t timeUs;
383    CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
384    native_window_set_buffers_timestamp(anw, timeUs * 1000);
385    status_t err = anw->queueBuffer(anw, buffer->graphicBuffer().get(), -1);
386    if (err != 0) {
387        ALOGE("queueBuffer failed with error %s (%d)", strerror(-err), -err);
388        return;
389    }
390
391    sp<MetaData> metaData = buffer->meta_data();
392    metaData->setInt32(kKeyRendered, 1);
393}
394
395void NativeWindowRenderer::queueExternalBuffer(ANativeWindow* anw,
396    MediaBuffer* buffer, int width, int height) {
397    native_window_set_buffers_geometry(anw, width, height,
398            HAL_PIXEL_FORMAT_YV12);
399    native_window_set_usage(anw, GRALLOC_USAGE_SW_WRITE_OFTEN);
400
401    ANativeWindowBuffer* anb;
402    CHECK(NO_ERROR == native_window_dequeue_buffer_and_wait(anw, &anb));
403    CHECK(anb != NULL);
404
405    // Copy the buffer
406    uint8_t* img = NULL;
407    sp<GraphicBuffer> buf(new GraphicBuffer(anb, false));
408    buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img));
409    copyI420Buffer(buffer, img, width, height, buf->getStride());
410    buf->unlock();
411    CHECK(NO_ERROR == anw->queueBuffer(anw, buf->getNativeBuffer(), -1));
412}
413
414void NativeWindowRenderer::copyI420Buffer(MediaBuffer* src, uint8_t* dst,
415        int srcWidth, int srcHeight, int stride) {
416    int strideUV = (stride / 2 + 0xf) & ~0xf;
417    uint8_t* p = (uint8_t*)src->data() + src->range_offset();
418    // Y
419    for (int i = srcHeight; i > 0; i--) {
420        memcpy(dst, p, srcWidth);
421        dst += stride;
422        p += srcWidth;
423    }
424    // The src is I420, the dst is YV12.
425    // U
426    p += srcWidth * srcHeight / 4;
427    for (int i = srcHeight / 2; i > 0; i--) {
428        memcpy(dst, p, srcWidth / 2);
429        dst += strideUV;
430        p += srcWidth / 2;
431    }
432    // V
433    p -= srcWidth * srcHeight / 2;
434    for (int i = srcHeight / 2; i > 0; i--) {
435        memcpy(dst, p, srcWidth / 2);
436        dst += strideUV;
437        p += srcWidth / 2;
438    }
439}
440
441void NativeWindowRenderer::updateProgramAndHandle(uint32_t videoEffect) {
442    if (mLastVideoEffect == videoEffect) {
443        return;
444    }
445
446    mLastVideoEffect = videoEffect;
447    int i;
448    switch (mLastVideoEffect) {
449        case VIDEO_EFFECT_NONE:
450            i = 0;
451            break;
452        case VIDEO_EFFECT_SEPIA:
453            i = 1;
454            break;
455        case VIDEO_EFFECT_NEGATIVE:
456            i = 2;
457            break;
458        case VIDEO_EFFECT_GRADIENT:
459            i = 3;
460            break;
461        default:
462            i = 0;
463            break;
464    }
465    glUseProgram(mProgram[i]);
466    CHECK_GL_ERROR;
467
468    mPositionHandle = glGetAttribLocation(mProgram[i], "vPosition");
469    mTexPosHandle = glGetAttribLocation(mProgram[i], "vTexPos");
470    mTexMatrixHandle = glGetUniformLocation(mProgram[i], "texMatrix");
471    CHECK_GL_ERROR;
472}
473
474void NativeWindowRenderer::calculatePositionCoordinates(
475        M4xVSS_MediaRendering renderingMode, int srcWidth, int srcHeight) {
476    float x, y;
477    switch (renderingMode) {
478        case M4xVSS_kResizing:
479        default:
480            x = 1;
481            y = 1;
482            break;
483        case M4xVSS_kCropping:
484            x = float(srcWidth) / mDstWidth;
485            y = float(srcHeight) / mDstHeight;
486            // Make the smaller side 1
487            if (x > y) {
488                x /= y;
489                y = 1;
490            } else {
491                y /= x;
492                x = 1;
493            }
494            break;
495        case M4xVSS_kBlackBorders:
496            x = float(srcWidth) / mDstWidth;
497            y = float(srcHeight) / mDstHeight;
498            // Make the larger side 1
499            if (x > y) {
500                y /= x;
501                x = 1;
502            } else {
503                x /= y;
504                y = 1;
505            }
506            break;
507    }
508
509    mPositionCoordinates[0] = -x;
510    mPositionCoordinates[1] = y;
511    mPositionCoordinates[2] = -x;
512    mPositionCoordinates[3] = -y;
513    mPositionCoordinates[4] = x;
514    mPositionCoordinates[5] = -y;
515    mPositionCoordinates[6] = x;
516    mPositionCoordinates[7] = y;
517}
518
519//
520//  The functions below run in other threads.
521//
522
523void NativeWindowRenderer::startRequest(int cmd) {
524    mLock.lock();
525    while (mThreadCmd != CMD_IDLE) {
526        mCond.wait(mLock);
527    }
528    mThreadCmd = cmd;
529}
530
531void NativeWindowRenderer::sendRequest() {
532    mCond.broadcast();
533    while (mThreadCmd != CMD_IDLE) {
534        mCond.wait(mLock);
535    }
536    mLock.unlock();
537}
538
539RenderInput* NativeWindowRenderer::createRenderInput() {
540    ALOGD("new render input %d", mNextTextureId);
541    RenderInput* input = new RenderInput(this, mNextTextureId);
542
543    startRequest(CMD_RESERVE_TEXTURE);
544    mThreadTextureId = mNextTextureId;
545    sendRequest();
546
547    mNextTextureId++;
548    mActiveInputs++;
549    return input;
550}
551
552void NativeWindowRenderer::destroyRenderInput(RenderInput* input) {
553    ALOGD("destroy render input %d", input->mTextureId);
554    GLuint textureId = input->mTextureId;
555    delete input;
556
557    startRequest(CMD_DELETE_TEXTURE);
558    mThreadTextureId = textureId;
559    sendRequest();
560
561    mActiveInputs--;
562}
563
564//
565//  RenderInput
566//
567
568RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId)
569    : mRenderer(renderer)
570    , mTextureId(textureId) {
571    mST = new SurfaceTexture(mTextureId);
572    mSTC = new SurfaceTextureClient(mST);
573    native_window_connect(mSTC.get(), NATIVE_WINDOW_API_MEDIA);
574}
575
576RenderInput::~RenderInput() {
577}
578
579ANativeWindow* RenderInput::getTargetWindow() {
580    return mSTC.get();
581}
582
583void RenderInput::updateVideoSize(sp<MetaData> meta) {
584    CHECK(meta->findInt32(kKeyWidth, &mWidth));
585    CHECK(meta->findInt32(kKeyHeight, &mHeight));
586
587    int left, top, right, bottom;
588    if (meta->findRect(kKeyCropRect, &left, &top, &right, &bottom)) {
589        mWidth = right - left + 1;
590        mHeight = bottom - top + 1;
591    }
592
593    // If rotation degrees is 90 or 270, swap width and height
594    // (mWidth and mHeight are the _rotated_ source rectangle).
595    int32_t rotationDegrees;
596    if (!meta->findInt32(kKeyRotation, &rotationDegrees)) {
597        rotationDegrees = 0;
598    }
599
600    if (rotationDegrees == 90 || rotationDegrees == 270) {
601        int tmp = mWidth;
602        mWidth = mHeight;
603        mHeight = tmp;
604    }
605}
606
607void RenderInput::render(MediaBuffer* buffer, uint32_t videoEffect,
608        M4xVSS_MediaRendering renderingMode, bool isExternalBuffer) {
609    mVideoEffect = videoEffect;
610    mRenderingMode = renderingMode;
611    mIsExternalBuffer = isExternalBuffer;
612    mBuffer = buffer;
613
614    mRenderer->startRequest(NativeWindowRenderer::CMD_RENDER_INPUT);
615    mRenderer->mThreadRenderInput = this;
616    mRenderer->sendRequest();
617}
618
619}  // namespace android
620