143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang/* 243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * Copyright (C) 2011 The Android Open Source Project 343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * 443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * Licensed under the Apache License, Version 2.0 (the "License"); 543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * you may not use this file except in compliance with the License. 643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * You may obtain a copy of the License at 743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * 843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * http://www.apache.org/licenses/LICENSE-2.0 943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * 1043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * Unless required by applicable law or agreed to in writing, software 1143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * distributed under the License is distributed on an "AS IS" BASIS, 1243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * See the License for the specific language governing permissions and 1443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang * limitations under the License. 1543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang */ 1643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 1743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#ifndef NATIVE_WINDOW_RENDERER_H_ 1843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#define NATIVE_WINDOW_RENDERER_H_ 1943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 2043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#include <EGL/egl.h> 2143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#include <GLES2/gl2.h> 22c1e3ed15d86337361c11fc4bb425ae252eceb946Glenn Kasten#include <media/stagefright/MediaBuffer.h> 23c1e3ed15d86337361c11fc4bb425ae252eceb946Glenn Kasten#include <media/stagefright/MetaData.h> 2443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#include <utils/RefBase.h> 2543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#include <utils/threads.h> 2643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 2743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#include "M4xVSS_API.h" 2843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 2943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// The NativeWindowRenderer draws video frames stored in MediaBuffers to 3043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// an ANativeWindow. It can apply "rendering mode" and color effects to 3143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// the frames. "Rendering mode" is the option to do resizing, cropping, 3243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// or black-bordering when the source and destination aspect ratio are 3343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// different. Color effects include sepia, negative, and gradient. 3443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// 3543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// The input to NativeWindowRenderer is provided by the RenderInput class, 3643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// and there can be multiple active RenderInput at the same time. Although 3743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// we only expect that happens briefly when one clip is about to finish 3843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// and the next clip is about to start. 3943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// 4043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// We allocate a SurfaceTexture for each RenderInput and the user can use 4143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// the getTargetWindow() function to get the corresponding ANativeWindow 4243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// for that SurfaceTexture. The intention is that the user can pass that 4343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// ANativeWindow to OMXCodec::Create() so the codec can decode directly 4443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang// to buffers provided by the texture. 4543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 4643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changnamespace android { 4743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 4843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changclass SurfaceTexture; 4943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changclass SurfaceTextureClient; 5043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changclass RenderInput; 5143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 5243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changclass NativeWindowRenderer { 5343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changpublic: 5443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang NativeWindowRenderer(sp<ANativeWindow> nativeWindow, int width, int height); 5543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang ~NativeWindowRenderer(); 5643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 5743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang RenderInput* createRenderInput(); 5843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void destroyRenderInput(RenderInput* input); 5943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 6043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changprivate: 6143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // No copy constructor and assignment 6243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang NativeWindowRenderer(const NativeWindowRenderer &); 6343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang NativeWindowRenderer &operator=(const NativeWindowRenderer &); 6443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 6543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Initialization and finialization 6643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void initializeEGL(); 6743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void terminateEGL(); 6843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void createPrograms(); 6943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void createProgram( 7043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm); 7143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void loadShader( 7243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLenum shaderType, const char* pSource, GLuint* outShader); 7343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 7443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // These functions are executed every frame. 7543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void render(RenderInput* input); 7643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void queueInternalBuffer(ANativeWindow* anw, MediaBuffer* buffer); 7743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void queueExternalBuffer(ANativeWindow* anw, MediaBuffer* buffer, 7843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int width, int height); 792aa01fd002bba1dde45791c1138c1f71a8d0aa53Chih-Chung Chang void copyI420Buffer(MediaBuffer* src, uint8_t* dst, 8043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int srcWidth, int srcHeight, int stride); 8143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void updateProgramAndHandle(uint32_t videoEffect); 8243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void calculatePositionCoordinates(M4xVSS_MediaRendering renderingMode, 8343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int srcWidth, int srcHeight); 8443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 8543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // These variables are initialized once and doesn't change afterwards. 8643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang sp<ANativeWindow> mNativeWindow; 8743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int mDstWidth, mDstHeight; 8843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EGLDisplay mEglDisplay; 8943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EGLSurface mEglSurface; 9043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EGLContext mEglContext; 9143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang enum { 9243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EFFECT_NORMAL, 9343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EFFECT_SEPIA, 9443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EFFECT_NEGATIVE, 9543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang EFFECT_GRADIENT, 9643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang NUMBER_OF_EFFECTS 9743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang }; 9843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLuint mProgram[NUMBER_OF_EFFECTS]; 9943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 10043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // We use one shader program for each effect. mLastVideoEffect remembers 10143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // the program used for the last frame. when the effect used changes, 10243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // we change the program used and update the handles. 10343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang uint32_t mLastVideoEffect; 10443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLint mPositionHandle; 10543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLint mTexPosHandle; 10643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLint mTexMatrixHandle; 10743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 10843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // This is the vertex coordinates used for the frame texture. 10943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // It's calculated according the the rendering mode and the source and 11043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // destination aspect ratio. 11143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLfloat mPositionCoordinates[8]; 11243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 11343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // We use a different GL id for each SurfaceTexture. 11443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLuint mNextTextureId; 11543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 11643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Number of existing RenderInputs, just for debugging. 11743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int mActiveInputs; 11843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 11943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // The GL thread functions 12043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang static int threadStart(void* self); 12143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void glThread(); 12243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 12343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // These variables are used to communicate between the GL thread and 12443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // other threads. 12543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang Mutex mLock; 12643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang Condition mCond; 12743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang enum { 12843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang CMD_IDLE, 12943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang CMD_RENDER_INPUT, 13043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang CMD_RESERVE_TEXTURE, 13143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang CMD_DELETE_TEXTURE, 13243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang CMD_QUIT, 13343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang }; 13443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int mThreadCmd; 13543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang RenderInput* mThreadRenderInput; 13643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLuint mThreadTextureId; 13743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 13843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // These functions are used to send commands to the GL thread. 13943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // sendRequest() also waits for the GL thread acknowledges the 14043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // command is finished. 14143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void startRequest(int cmd); 14243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void sendRequest(); 14343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 14443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang friend class RenderInput; 14543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang}; 14643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 14743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changclass RenderInput { 14843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changpublic: 14943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Returns the ANativeWindow corresponds to the SurfaceTexture. 15043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang ANativeWindow* getTargetWindow(); 15143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 15243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Updates video frame size from the MediaSource's metadata. Specifically 15343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // we look for kKeyWidth, kKeyHeight, and (optionally) kKeyCropRect. 15443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void updateVideoSize(sp<MetaData> meta); 15543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 15643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Renders the buffer with the given video effect and rending mode. 15743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // The video effets are defined in VideoEditorTools.h 15843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // Set isExternalBuffer to true only when the buffer given is not 15943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // provided by the SurfaceTexture. 16043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang void render(MediaBuffer *buffer, uint32_t videoEffect, 16143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang M4xVSS_MediaRendering renderingMode, bool isExternalBuffer); 16243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Changprivate: 16343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang RenderInput(NativeWindowRenderer* renderer, GLuint textureId); 16443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang ~RenderInput(); 16543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang NativeWindowRenderer* mRenderer; 16643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang GLuint mTextureId; 16743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang sp<SurfaceTexture> mST; 16843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang sp<SurfaceTextureClient> mSTC; 16943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang int mWidth, mHeight; 17043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 17143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang // These are only valid during render() calls 17243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang uint32_t mVideoEffect; 17343fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang M4xVSS_MediaRendering mRenderingMode; 17443fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang bool mIsExternalBuffer; 17543fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang MediaBuffer* mBuffer; 17643fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 17743fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang friend class NativeWindowRenderer; 17843fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang}; 17943fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 18043fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang} // namespace android 18143fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang 18243fcc396614a587851e2b7c4cea2876ec58b8648Chih-Chung Chang#endif // NATIVE_WINDOW_RENDERER_H_ 183