150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org/* 250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * libjingle 350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Copyright 2014, Google Inc. 450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Redistribution and use in source and binary forms, with or without 650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * modification, are permitted provided that the following conditions are met: 750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 1. Redistributions of source code must retain the above copyright notice, 950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * this list of conditions and the following disclaimer. 1050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 2. Redistributions in binary form must reproduce the above copyright notice, 1150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * this list of conditions and the following disclaimer in the documentation 1250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * and/or other materials provided with the distribution. 1350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 3. The name of the author may not be used to endorse or promote products 1450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * derived from this software without specific prior written permission. 1550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * 1650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 1750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 1850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 1950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 2150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 2250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 2350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 2450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 2550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org */ 2750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 2850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgpackage org.webrtc; 2950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 3050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.nio.ByteBuffer; 3150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.nio.ByteOrder; 3250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.nio.FloatBuffer; 3350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.util.ArrayList; 3450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.util.concurrent.CountDownLatch; 3550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport java.util.concurrent.LinkedBlockingQueue; 3650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 3750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport javax.microedition.khronos.egl.EGLConfig; 3850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport javax.microedition.khronos.opengles.GL10; 3950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 40a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.orgimport android.graphics.SurfaceTexture; 41a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.orgimport android.opengl.EGL14; 42a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.orgimport android.opengl.EGLContext; 43a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.orgimport android.opengl.GLES11Ext; 4450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport android.opengl.GLES20; 4550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport android.opengl.GLSurfaceView; 4650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport android.util.Log; 4750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 4850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgimport org.webrtc.VideoRenderer.I420Frame; 4950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 5050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org/** 5150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Efficiently renders YUV frames using the GPU for CSC. 5250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Clients will want first to call setView() to pass GLSurfaceView 5350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * and then for each video stream either create instance of VideoRenderer using 5450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * createGui() call or VideoRenderer.Callbacks interface using create() call. 5550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Only one instance of the class can be created. 5650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org */ 5750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.orgpublic class VideoRendererGui implements GLSurfaceView.Renderer { 5850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static VideoRendererGui instance = null; 5950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static final String TAG = "VideoRendererGui"; 6050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private GLSurfaceView surface; 61a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private static EGLContext eglContext = null; 6250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Indicates if SurfaceView.Renderer.onSurfaceCreated was called. 6350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // If true then for every newly created yuv image renderer createTexture() 6450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // should be called. The variable is accessed on multiple threads and 6550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // all accesses are synchronized on yuvImageRenderers' object lock. 6650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private boolean onSurfaceCreatedCalled; 6756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int screenWidth; 6856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int screenHeight; 6950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // List of yuv renderers. 7050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private ArrayList<YuvImageRenderer> yuvImageRenderers; 71a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int yuvProgram; 72a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int oesProgram; 7356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Types of video scaling: 7456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by 7556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // maintaining the aspect ratio (black borders may be displayed). 7656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by 7756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // maintaining the aspect ratio. Some portion of the video frame may be 7856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // clipped. 7956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // SCALE_FILL - video frame is scaled to to fill the size of the view. Video 8056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // aspect ratio is changed if necessary. 8156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private static enum ScalingType 8256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_FILL }; 8350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 8450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private final String VERTEX_SHADER_STRING = 8550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "varying vec2 interp_tc;\n" + 8650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "attribute vec4 in_pos;\n" + 8750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "attribute vec2 in_tc;\n" + 8850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "\n" + 8950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "void main() {\n" + 9050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " gl_Position = in_pos;\n" + 9150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " interp_tc = in_tc;\n" + 9250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "}\n"; 9350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 94a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private final String YUV_FRAGMENT_SHADER_STRING = 9550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "precision mediump float;\n" + 9650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "varying vec2 interp_tc;\n" + 9750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "\n" + 9850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "uniform sampler2D y_tex;\n" + 9950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "uniform sampler2D u_tex;\n" + 10050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "uniform sampler2D v_tex;\n" + 10150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "\n" + 10250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "void main() {\n" + 10350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // CSC according to http://www.fourcc.org/fccyvrgb.php 10450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " float y = texture2D(y_tex, interp_tc).r;\n" + 10550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" + 10650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" + 10750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " gl_FragColor = vec4(y + 1.403 * v, " + 10850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " y - 0.344 * u - 0.714 * v, " + 10950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " y + 1.77 * u, 1);\n" + 11050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "}\n"; 11150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 112a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 113a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private static final String OES_FRAGMENT_SHADER_STRING = 114a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "#extension GL_OES_EGL_image_external : require\n" + 115a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "precision mediump float;\n" + 116a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "varying vec2 interp_tc;\n" + 117a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "\n" + 118a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "uniform samplerExternalOES oes_tex;\n" + 119a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "\n" + 120a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "void main() {\n" + 121a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org " gl_FragColor = texture2D(oes_tex, interp_tc);\n" + 122a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org "}\n"; 123a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 124a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 12550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private VideoRendererGui(GLSurfaceView surface) { 12650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org this.surface = surface; 12750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Create an OpenGL ES 2.0 context. 12850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org surface.setPreserveEGLContextOnPause(true); 12950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org surface.setEGLContextClientVersion(2); 13050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org surface.setRenderer(this); 13150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org surface.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); 13250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 13350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org yuvImageRenderers = new ArrayList<YuvImageRenderer>(); 13450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 13550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 13650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Poor-man's assert(): die with |msg| unless |condition| is true. 13750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static void abortUnless(boolean condition, String msg) { 13850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (!condition) { 13950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org throw new RuntimeException(msg); 14050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 14150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 14250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 14350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Assert that no OpenGL ES 2.0 error has been raised. 14450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static void checkNoGLES2Error() { 14550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org int error = GLES20.glGetError(); 14650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org abortUnless(error == GLES20.GL_NO_ERROR, "GLES20 error: " + error); 14750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 14850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 14950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Wrap a float[] in a direct FloatBuffer using native byte order. 15050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static FloatBuffer directNativeFloatBuffer(float[] array) { 15150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org FloatBuffer buffer = ByteBuffer.allocateDirect(array.length * 4).order( 15250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org ByteOrder.nativeOrder()).asFloatBuffer(); 15350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org buffer.put(array); 15450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org buffer.flip(); 15550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return buffer; 15650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 15750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 158a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int loadShader(int shaderType, String source) { 15950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org int[] result = new int[] { 16050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.GL_FALSE 16150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org }; 162a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int shader = GLES20.glCreateShader(shaderType); 16350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glShaderSource(shader, source); 16450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glCompileShader(shader); 16550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0); 166a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (result[0] != GLES20.GL_TRUE) { 167a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.e(TAG, "Could not compile shader " + shaderType + ":" + 168a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glGetShaderInfoLog(shader)); 169a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException(GLES20.glGetShaderInfoLog(shader)); 170a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 171a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org checkNoGLES2Error(); 172a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org return shader; 173a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org} 174a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 17550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 176a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int createProgram(String vertexSource, String fragmentSource) { 177a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); 178a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); 179a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int program = GLES20.glCreateProgram(); 180a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (program == 0) { 181a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException("Could not create program"); 182a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 183a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glAttachShader(program, vertexShader); 184a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glAttachShader(program, fragmentShader); 185a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glLinkProgram(program); 186a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int[] linkStatus = new int[] { 187a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.GL_FALSE 188a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org }; 189a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); 190a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (linkStatus[0] != GLES20.GL_TRUE) { 191a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.e(TAG, "Could not link program: " + 192a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glGetProgramInfoLog(program)); 193a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException(GLES20.glGetProgramInfoLog(program)); 194a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 19550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org checkNoGLES2Error(); 196a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org return program; 197a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org} 19850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 19950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org /** 20050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Class used to display stream of YUV420 frames at particular location 20150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * on a screen. New video frames are sent to display using renderFrame() 20250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * call. 20350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org */ 20450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private static class YuvImageRenderer implements VideoRenderer.Callbacks { 20550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private GLSurfaceView surface; 206a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int id; 207a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int yuvProgram; 208a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int oesProgram; 20950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private int[] yuvTextures = { -1, -1, -1 }; 210a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private int oesTexture = -1; 211a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private float[] stMatrix = new float[16]; 21250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 21350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Render frame queue - accessed by two threads. renderFrame() call does 21450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // an offer (writing I420Frame to render) and early-returns (recording 21550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // a dropped frame) if that queue is full. draw() call does a peek(), 21650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // copies frame to texture and then removes it from a queue using poll(). 21750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org LinkedBlockingQueue<I420Frame> frameToRenderQueue; 21850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Local copy of incoming video frame. 219a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private I420Frame yuvFrameToRender; 220a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private I420Frame textureFrameToRender; 221a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Type of video frame used for recent frame rendering. 222a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private static enum RendererType { RENDERER_YUV, RENDERER_TEXTURE }; 223a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private RendererType rendererType; 22456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private ScalingType scalingType; 225a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Flag if renderFrame() was ever called. 22650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org boolean seenFrame; 22750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Total number of video frames received in renderFrame() call. 22850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private int framesReceived; 22950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Number of video frames dropped by renderFrame() because previous 23050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // frame has not been rendered yet. 23150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private int framesDropped; 23250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Number of rendered video frames. 23350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private int framesRendered; 23450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Time in ns when the first video frame was rendered. 23550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private long startTimeNs = -1; 23650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Time in ns spent in draw() function. 23750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private long drawTimeNs; 23850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Time in ns spent in renderFrame() function - including copying frame 239a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // data to rendering planes. 24050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private long copyTimeNs; 24156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Texture vertices. 24256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texLeft; 24356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texRight; 24456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texTop; 24556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texBottom; 24656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private FloatBuffer textureVertices; 24756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Texture UV coordinates offsets. 24856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texOffsetU; 24956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private float texOffsetV; 25056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private FloatBuffer textureCoords; 25156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Flag if texture vertices or coordinates update is needed. 25256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private boolean updateTextureProperties; 25356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Viewport dimensions. 25456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int screenWidth; 25556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int screenHeight; 25656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Video dimension. 25756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int videoWidth; 25856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private int videoHeight; 25950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 26050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private YuvImageRenderer( 261a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLSurfaceView surface, int id, 26256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org int x, int y, int width, int height, 26356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org ScalingType scalingType) { 264a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "YuvImageRenderer.Create id: " + id); 26550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org this.surface = surface; 266a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org this.id = id; 26756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org this.scalingType = scalingType; 26850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1); 26950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Create texture vertices. 27056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft = (x - 50) / 50.0f; 27156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texTop = (50 - y) / 50.0f; 27256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight = Math.min(1.0f, (x + width - 50) / 50.0f); 27356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texBottom = Math.max(-1.0f, (50 - y - height) / 50.0f); 27450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org float textureVeticesFloat[] = new float[] { 27556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft, texTop, 27656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft, texBottom, 27756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight, texTop, 27856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight, texBottom 27950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org }; 28050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org textureVertices = directNativeFloatBuffer(textureVeticesFloat); 28156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Create texture UV coordinates. 28256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU = 0; 28356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetV = 0; 28456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float textureCoordinatesFloat[] = new float[] { 28556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU, texOffsetV, // left top 28656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU, 1.0f - texOffsetV, // left bottom 28756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 1.0f - texOffsetU, texOffsetV, // right top 28856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 1.0f - texOffsetU, 1.0f - texOffsetV // right bottom 28956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org }; 29056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org textureCoords = directNativeFloatBuffer(textureCoordinatesFloat); 29156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org updateTextureProperties = false; 29250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 29350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 294a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org private void createTextures(int yuvProgram, int oesProgram) { 295a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, " YuvImageRenderer.createTextures " + id + " on GL thread:" + 296a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Thread.currentThread().getId()); 297a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org this.yuvProgram = yuvProgram; 298a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org this.oesProgram = oesProgram; 29950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 300a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Generate 3 texture ids for Y/U/V and place them into |yuvTextures|. 30150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glGenTextures(3, yuvTextures, 0); 30250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org for (int i = 0; i < 3; i++) { 30350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 30450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); 30550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, 30650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 128, 128, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null); 30750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 30850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); 30950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 31050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); 31150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 31250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); 31350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 31450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); 31550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 31650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org checkNoGLES2Error(); 31750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 31850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 31956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org private void checkAdjustTextureCoords() { 32056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (!updateTextureProperties || 32156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org scalingType == ScalingType.SCALE_FILL) { 32256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org return; 32356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 32456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Re - calculate texture vertices to preserve video aspect ratio. 32556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float texRight = this.texRight; 32656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float texLeft = this.texLeft; 32756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float texTop = this.texTop; 32856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float texBottom = this.texBottom; 32956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float displayWidth = (texRight - texLeft) * screenWidth / 2; 33056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float displayHeight = (texTop - texBottom) * screenHeight / 2; 33156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (displayWidth > 1 && displayHeight > 1 && 33256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org videoWidth > 1 && videoHeight > 1) { 33356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float displayAspectRatio = displayWidth / displayHeight; 33456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float videoAspectRatio = (float)videoWidth / videoHeight; 33556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (scalingType == ScalingType.SCALE_ASPECT_FIT) { 33656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Need to re-adjust vertices width or height to match video AR. 33756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (displayAspectRatio > videoAspectRatio) { 33856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float deltaX = (displayWidth - videoAspectRatio * displayHeight) / 33956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org instance.screenWidth; 34056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight -= deltaX; 34156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft += deltaX; 34256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } else { 34356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float deltaY = (displayHeight - displayWidth / videoAspectRatio) / 34456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org instance.screenHeight; 34556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texTop -= deltaY; 34656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texBottom += deltaY; 34756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 34856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Re-allocate vertices buffer to adjust to video aspect ratio. 34956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float textureVeticesFloat[] = new float[] { 35056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft, texTop, 35156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texLeft, texBottom, 35256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight, texTop, 35356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texRight, texBottom 35456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org }; 35556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org textureVertices = directNativeFloatBuffer(textureVeticesFloat); 35656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 35756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (scalingType == ScalingType.SCALE_ASPECT_FILL) { 35856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Need to re-adjust UV coordinates to match display AR. 35956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if (displayAspectRatio > videoAspectRatio) { 36056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetV = (1.0f - videoAspectRatio / displayAspectRatio) / 2.0f; 36156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } else { 36256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU = (1.0f - displayAspectRatio / videoAspectRatio) / 2.0f; 36356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 36456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Re-allocate coordinates buffer to adjust to display aspect ratio. 36556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org float textureCoordinatesFloat[] = new float[] { 36656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU, texOffsetV, // left top 36756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org texOffsetU, 1.0f - texOffsetV, // left bottom 36856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 1.0f - texOffsetU, texOffsetV, // right top 36956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 1.0f - texOffsetU, 1.0f - texOffsetV // right bottom 37056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org }; 37156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org textureCoords = directNativeFloatBuffer(textureCoordinatesFloat); 37256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 37356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 37456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org updateTextureProperties = false; 37556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 37656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 37750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private void draw() { 37850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (!seenFrame) { 37950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // No frame received yet - nothing to render. 38050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return; 38150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 38256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // Check if texture vertices/coordinates adjustment is required when 38356dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org // screen orientation changes or video frame size changes. 38456dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org checkAdjustTextureCoords(); 38556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 386a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org long now = System.nanoTime(); 387a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 38850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org I420Frame frameFromQueue; 38950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org synchronized (frameToRenderQueue) { 39050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org frameFromQueue = frameToRenderQueue.peek(); 39150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (frameFromQueue != null && startTimeNs == -1) { 39250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org startTimeNs = now; 39350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 394a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 395a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (rendererType == RendererType.RENDERER_YUV) { 396a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // YUV textures rendering. 397a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glUseProgram(yuvProgram); 398a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 399a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org for (int i = 0; i < 3; ++i) { 400a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 401a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); 402a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (frameFromQueue != null) { 403a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int w = (i == 0) ? 404a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frameFromQueue.width : frameFromQueue.width / 2; 405a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int h = (i == 0) ? 406a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frameFromQueue.height : frameFromQueue.height / 2; 407a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, 408a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org w, h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, 409a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frameFromQueue.yuvPlanes[i]); 410a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 411a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 412a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } else { 413a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // External texture rendering. 414a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glUseProgram(oesProgram); 415a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 41650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (frameFromQueue != null) { 417a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org oesTexture = frameFromQueue.textureId; 418a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (frameFromQueue.textureObject instanceof SurfaceTexture) { 419a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org SurfaceTexture surfaceTexture = 420a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org (SurfaceTexture) frameFromQueue.textureObject; 421a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org surfaceTexture.updateTexImage(); 422a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org surfaceTexture.getTransformMatrix(stMatrix); 423a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 42450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 425a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 426a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTexture); 42750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 428a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 42950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (frameFromQueue != null) { 43050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org frameToRenderQueue.poll(); 43150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 43250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 433a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 434a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (rendererType == RendererType.RENDERER_YUV) { 435a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glUniform1i(GLES20.glGetUniformLocation(yuvProgram, "y_tex"), 0); 436a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glUniform1i(GLES20.glGetUniformLocation(yuvProgram, "u_tex"), 1); 437a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org GLES20.glUniform1i(GLES20.glGetUniformLocation(yuvProgram, "v_tex"), 2); 438a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 439a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 440a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int posLocation = GLES20.glGetAttribLocation(yuvProgram, "in_pos"); 441a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (posLocation == -1) { 442a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException("Could not get attrib location for in_pos"); 443a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 44450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glEnableVertexAttribArray(posLocation); 44550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glVertexAttribPointer( 44650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org posLocation, 2, GLES20.GL_FLOAT, false, 0, textureVertices); 44750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 448a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int texLocation = GLES20.glGetAttribLocation(yuvProgram, "in_tc"); 449a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (texLocation == -1) { 450a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException("Could not get attrib location for in_tc"); 451a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 45250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glEnableVertexAttribArray(texLocation); 45350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glVertexAttribPointer( 45450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords); 45550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 45650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); 45750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 45850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glDisableVertexAttribArray(posLocation); 45950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glDisableVertexAttribArray(texLocation); 46050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 46150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org checkNoGLES2Error(); 46250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 46350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (frameFromQueue != null) { 46450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org framesRendered++; 46550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org drawTimeNs += (System.nanoTime() - now); 46656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org if ((framesRendered % 150) == 0) { 46750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org logStatistics(); 46850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 46950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 47050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 47150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 47250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org private void logStatistics() { 47350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org long timeSinceFirstFrameNs = System.nanoTime() - startTimeNs; 474a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "ID: " + id + ". Type: " + rendererType + 475a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org ". Frames received: " + framesReceived + 476a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org ". Dropped: " + framesDropped + ". Rendered: " + framesRendered); 47750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (framesReceived > 0 && framesRendered > 0) { 478a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "Duration: " + (int)(timeSinceFirstFrameNs / 1e6) + 47950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org " ms. FPS: " + (float)framesRendered * 1e9 / timeSinceFirstFrameNs); 480a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "Draw time: " + 48150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org (int) (drawTimeNs / (1000 * framesRendered)) + " us. Copy time: " + 48250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org (int) (copyTimeNs / (1000 * framesReceived)) + " us"); 48350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 48450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 48550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 48656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org public void setScreenSize(final int screenWidth, final int screenHeight) { 48756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org this.screenWidth = screenWidth; 48856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org this.screenHeight = screenHeight; 48956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org updateTextureProperties = true; 49056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 49156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org 49250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org @Override 49350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public void setSize(final int width, final int height) { 494a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " + 495a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org width + " x " + height); 49656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org videoWidth = width; 49756dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org videoHeight = height; 49850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org int[] strides = { width, width / 2, width / 2 }; 49950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Frame re-allocation need to be synchronized with copying 50050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // frame to textures in draw() function to avoid re-allocating 50150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // the frame while it is being copied. 50250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org synchronized (frameToRenderQueue) { 503a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Clear rendering queue. 50450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org frameToRenderQueue.poll(); 505a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Re-allocate / allocate the frame. 506a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org yuvFrameToRender = new I420Frame(width, height, strides, null); 507a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org textureFrameToRender = new I420Frame(width, height, null, -1); 50856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org updateTextureProperties = true; 50950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 51050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 51150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 51250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org @Override 51350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public synchronized void renderFrame(I420Frame frame) { 51450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org long now = System.nanoTime(); 51550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org framesReceived++; 51650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Skip rendering of this frame if setSize() was not called. 517a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (yuvFrameToRender == null || textureFrameToRender == null) { 51850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org framesDropped++; 51950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return; 52050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 521a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Check input frame parameters. 522a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (frame.yuvFrame) { 523a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (!(frame.yuvStrides[0] == frame.width && 524a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frame.yuvStrides[1] == frame.width / 2 && 525a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frame.yuvStrides[2] == frame.width / 2)) { 526a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.e(TAG, "Incorrect strides " + frame.yuvStrides[0] + ", " + 527a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frame.yuvStrides[1] + ", " + frame.yuvStrides[2]); 528a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org return; 529a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 530a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Check incoming frame dimensions. 531a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (frame.width != yuvFrameToRender.width || 532a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frame.height != yuvFrameToRender.height) { 533a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org throw new RuntimeException("Wrong frame size " + 534a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frame.width + " x " + frame.height); 535a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 53650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 53750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 53850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (frameToRenderQueue.size() > 0) { 53950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Skip rendering of this frame if previous frame was not rendered yet. 54050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org framesDropped++; 54150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return; 54250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 543a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 544a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Create a local copy of the frame. 545a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org if (frame.yuvFrame) { 546a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org yuvFrameToRender.copyFrom(frame); 547a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org rendererType = RendererType.RENDERER_YUV; 548a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frameToRenderQueue.offer(yuvFrameToRender); 549a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } else { 550a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org textureFrameToRender.copyFrom(frame); 551a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org rendererType = RendererType.RENDERER_TEXTURE; 552a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org frameToRenderQueue.offer(textureFrameToRender); 553a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 55450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org copyTimeNs += (System.nanoTime() - now); 55550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org seenFrame = true; 556a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 557a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Request rendering. 55850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org surface.requestRender(); 55950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 560a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 56150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 56250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 56350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org /** Passes GLSurfaceView to video renderer. */ 56450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public static void setView(GLSurfaceView surface) { 565a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "VideoRendererGui.setView"); 56650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org instance = new VideoRendererGui(surface); 56750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 56850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 569a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org public static EGLContext getEGLContext() { 570a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org return eglContext; 571a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 572a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 57350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org /** 57450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Creates VideoRenderer with top left corner at (x, y) and resolution 57550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * (width, height). All parameters are in percentage of screen resolution. 57650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org */ 57750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public static VideoRenderer createGui( 57850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org int x, int y, int width, int height) throws Exception { 57950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org YuvImageRenderer javaGuiRenderer = create(x, y, width, height); 58050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return new VideoRenderer(javaGuiRenderer); 58150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 58250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 583a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org public static VideoRenderer.Callbacks createGuiRenderer( 584a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org int x, int y, int width, int height) { 585a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org return create(x, y, width, height); 586a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org } 587a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org 58850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org /** 58950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * Creates VideoRenderer.Callbacks with top left corner at (x, y) and 59050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * resolution (width, height). All parameters are in percentage of 59150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org * screen resolution. 59250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org */ 59350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public static YuvImageRenderer create( 59450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org int x, int y, int width, int height) { 59550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Check display region parameters. 59650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (x < 0 || x > 100 || y < 0 || y > 100 || 59750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org width < 0 || width > 100 || height < 0 || height > 100 || 59850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org x + width > 100 || y + height > 100) { 59950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org throw new RuntimeException("Incorrect window parameters."); 60050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 60150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 60250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (instance == null) { 60350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org throw new RuntimeException( 60450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org "Attempt to create yuv renderer before setting GLSurfaceView"); 60550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 60650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org final YuvImageRenderer yuvImageRenderer = new YuvImageRenderer( 607a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org instance.surface, instance.yuvImageRenderers.size(), 60856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org x, y, width, height, ScalingType.SCALE_ASPECT_FIT); 60950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org synchronized (instance.yuvImageRenderers) { 61050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org if (instance.onSurfaceCreatedCalled) { 61150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // onSurfaceCreated has already been called for VideoRendererGui - 61250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // need to create texture for new image and add image to the 61350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // rendering list. 61450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org final CountDownLatch countDownLatch = new CountDownLatch(1); 61550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org instance.surface.queueEvent(new Runnable() { 61650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public void run() { 617a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org yuvImageRenderer.createTextures( 618a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org instance.yuvProgram, instance.oesProgram); 61956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org yuvImageRenderer.setScreenSize( 62056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org instance.screenWidth, instance.screenHeight); 62150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org countDownLatch.countDown(); 62250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 62350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org }); 62450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Wait for task completion. 62550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org try { 62650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org countDownLatch.await(); 62750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } catch (InterruptedException e) { 62850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org throw new RuntimeException(e); 62950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 63050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 63150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Add yuv renderer to rendering list. 63250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org instance.yuvImageRenderers.add(yuvImageRenderer); 63350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 63450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org return yuvImageRenderer; 63550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 63650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 63750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org @Override 63850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public void onSurfaceCreated(GL10 unused, EGLConfig config) { 639a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "VideoRendererGui.onSurfaceCreated"); 640a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Store render EGL context 641a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org eglContext = EGL14.eglGetCurrentContext(); 642a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "VideoRendererGui EGL Context: " + eglContext); 64350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 644a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org // Create YUV and OES programs. 645a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org yuvProgram = createProgram(VERTEX_SHADER_STRING, 646a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org YUV_FRAGMENT_SHADER_STRING); 647a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org oesProgram = createProgram(VERTEX_SHADER_STRING, 648a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org OES_FRAGMENT_SHADER_STRING); 64950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 65050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org synchronized (yuvImageRenderers) { 65150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org // Create textures for all images. 65250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) { 653a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org yuvImageRenderer.createTextures(yuvProgram, oesProgram); 65450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 65550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org onSurfaceCreatedCalled = true; 65650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 65750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org checkNoGLES2Error(); 65856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org GLES20.glClearColor(0.0f, 0.0f, 0.1f, 1.0f); 65950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 66050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 66150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org @Override 66250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public void onSurfaceChanged(GL10 unused, int width, int height) { 663a846c2043a8abdbc8fd1f7511d824e6a64a8baf5glaznev@webrtc.org Log.d(TAG, "VideoRendererGui.onSurfaceChanged: " + 66450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org width + " x " + height + " "); 66556dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org screenWidth = width; 66656dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org screenHeight = height; 66750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glViewport(0, 0, width, height); 66856dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org synchronized (yuvImageRenderers) { 66956dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) { 67056dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org yuvImageRenderer.setScreenSize(screenWidth, screenHeight); 67156dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 67256dcc5b4f1a06cc282500e6e9b80f71fbb88c115glaznev@webrtc.org } 67350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 67450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 67550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org @Override 67650af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org public void onDrawFrame(GL10 unused) { 67750af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 67850af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org synchronized (yuvImageRenderers) { 67950af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) { 68050af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org yuvImageRenderer.draw(); 68150af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 68250af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 68350af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org } 68450af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org 68550af306dbdc1248470aac424d889dde4f952625dglaznev@webrtc.org} 686