MyGLSurfaceView.java revision 3bbd3bb7fcb082eba313614f072c11c41b38ebfe 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.example.nativemedia;
18
19import android.graphics.SurfaceTexture;
20import android.util.Log;
21
22import java.nio.ByteBuffer;
23import java.nio.ByteOrder;
24import java.nio.FloatBuffer;
25
26import javax.microedition.khronos.egl.EGLConfig;
27import javax.microedition.khronos.opengles.GL10;
28
29import android.view.MotionEvent;
30import android.content.Context;
31
32import android.opengl.GLES20;
33import android.opengl.GLSurfaceView;
34import android.opengl.Matrix;
35
36import android.hardware.SensorManager;
37import android.hardware.SensorEvent;
38import android.hardware.SensorEventListener;
39import android.hardware.Sensor;
40
41// Remove once surfacetexture timestamps are in
42import java.lang.System;
43
44import android.util.AttributeSet;
45
46public class MyGLSurfaceView extends GLSurfaceView implements SensorEventListener {
47
48    public MyGLSurfaceView(Context context, AttributeSet attributeSet) {
49        super(context, attributeSet);
50        init(context);
51    }
52
53    public MyGLSurfaceView(Context context) {
54        super(context);
55        init(context);
56    }
57
58    private void init(Context context) {
59        setEGLContextClientVersion(2);
60        mRenderer = new MyRenderer(context);
61        setRenderer(mRenderer);
62
63        mSensorManager = (SensorManager)context.getSystemService(Context.SENSOR_SERVICE);
64        mAcceleration = mSensorManager.getDefaultSensor(Sensor.TYPE_LINEAR_ACCELERATION);
65    }
66
67    public boolean onTouchEvent(final MotionEvent event) {
68        queueEvent(new Runnable(){
69            public void run() {
70                mRenderer.setPosition(event.getX() / getWidth(),
71                                      event.getY() / getHeight());
72            }});
73        return true;
74    }
75
76    @Override
77    public void onPause() {
78        super.onPause();
79
80        mSensorManager.unregisterListener(this);
81    }
82
83    @Override
84    public void onResume() {
85
86        queueEvent(new Runnable() {
87            public void run() {
88                }});
89
90        mSensorManager.registerListener(this, mAcceleration, SensorManager.SENSOR_DELAY_GAME);
91        super.onResume();
92    }
93
94    public void onSensorChanged(SensorEvent event) {
95        if (event.sensor.getType() == Sensor.TYPE_LINEAR_ACCELERATION) {
96            final float[] accelerationVector = event.values;
97            queueEvent(new Runnable(){
98                    public void run() {
99                        mRenderer.setAcceleration(accelerationVector);
100                    }});
101        }
102    }
103
104    public void onAccuracyChanged(Sensor sensor, int accuracy) {
105        // Ignoring sensor accuracy changes.
106    }
107
108    MyRenderer mRenderer;
109
110    SensorManager mSensorManager;
111    Sensor mAcceleration;
112
113    public SurfaceTexture getSurfaceTexture()
114    {
115        return mRenderer.getSurfaceTexture();
116    }
117
118}
119
120class MyRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
121
122    public MyRenderer(Context context) {
123
124        mContext = context;
125
126        mTriangleVertices = ByteBuffer.allocateDirect(mTriangleVerticesData.length
127                * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
128        mTriangleVertices.put(mTriangleVerticesData).position(0);
129
130        Matrix.setIdentityM(mSTMatrix, 0);
131        Matrix.setIdentityM(mMMatrix, 0);
132
133        float[] defaultAcceleration = {0.f,0.f,0.f};
134        setAcceleration(defaultAcceleration);
135        mPos[0] = 0.f;
136        mPos[1] = 0.f;
137        mPos[2] = 0.f;
138        mVel[0] = 0.f;
139        mVel[1] = 0.f;
140        mVel[2] = 0.f;
141
142    }
143
144    /* The following set methods are not synchronized, so should only
145     * be called within the rendering thread context. Use GLSurfaceView.queueEvent for safe access.
146     */
147    public void setPosition(float x, float y) {
148        /* Map from screen (0,0)-(1,1) to scene coordinates */
149        mPos[0] = (x*2-1)*mRatio;
150        mPos[1] = (-y)*2+1;
151        mPos[2] = 0.f;
152        mVel[0] = 0;
153        mVel[1] = 0;
154        mVel[2] = 0;
155    }
156
157    public void setAcceleration(float[] accelerationVector) {
158        mGForce[0] = accelerationVector[0];
159        mGForce[1] = accelerationVector[1];
160        mGForce[2] = accelerationVector[2];
161    }
162
163    public void onDrawFrame(GL10 glUnused) {
164        synchronized(this) {
165            if (updateSurface) {
166                mSurface.updateTexImage();
167
168                mSurface.getTransformMatrix(mSTMatrix);
169                // Until timestamp support gets in (very soon)
170                //long timestamp = mSurface.getTimestamp();
171                long timestamp = System.nanoTime();
172                doPhysics(timestamp);
173
174                updateSurface = false;
175            }
176        }
177
178        // Ignore the passed-in GL10 interface, and use the GLES20
179        // class's static methods instead.
180        GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
181        GLES20.glUseProgram(mProgram);
182        checkGlError("glUseProgram");
183
184        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
185        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
186
187        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
188        GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
189                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
190        checkGlError("glVertexAttribPointer maPosition");
191        GLES20.glEnableVertexAttribArray(maPositionHandle);
192        checkGlError("glEnableVertexAttribArray maPositionHandle");
193
194        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
195        GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
196                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
197        checkGlError("glVertexAttribPointer maTextureHandle");
198        GLES20.glEnableVertexAttribArray(maTextureHandle);
199        checkGlError("glEnableVertexAttribArray maTextureHandle");
200
201        Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0);
202        Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0);
203
204        GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
205        GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
206        GLES20.glUniform1f(muCRatioHandle, mMediaPlayerRatio);
207
208        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
209        checkGlError("glDrawArrays");
210    }
211
212    public void onSurfaceChanged(GL10 glUnused, int width, int height) {
213        // Ignore the passed-in GL10 interface, and use the GLES20
214        // class's static methods instead.
215        GLES20.glViewport(0, 0, width, height);
216        mRatio = (float) width / height;
217        Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7);
218    }
219
220    public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
221        // Ignore the passed-in GL10 interface, and use the GLES20
222        // class's static methods instead.
223
224        /* Set up alpha blending and an Android background color */
225        GLES20.glEnable(GLES20.GL_BLEND);
226        GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
227        GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f);
228
229        /* Set up shaders and handles to their variables */
230        mProgram = createProgram(mVertexShader, mFragmentShader);
231        if (mProgram == 0) {
232            return;
233        }
234        maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
235        checkGlError("glGetAttribLocation aPosition");
236        if (maPositionHandle == -1) {
237            throw new RuntimeException("Could not get attrib location for aPosition");
238        }
239        maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
240        checkGlError("glGetAttribLocation aTextureCoord");
241        if (maTextureHandle == -1) {
242            throw new RuntimeException("Could not get attrib location for aTextureCoord");
243        }
244
245        muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
246        checkGlError("glGetUniformLocation uMVPMatrix");
247        if (muMVPMatrixHandle == -1) {
248            throw new RuntimeException("Could not get attrib location for uMVPMatrix");
249        }
250
251        muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
252        checkGlError("glGetUniformLocation uSTMatrix");
253        if (muMVPMatrixHandle == -1) {
254            throw new RuntimeException("Could not get attrib location for uSTMatrix");
255        }
256
257        muCRatioHandle = GLES20.glGetUniformLocation(mProgram, "uCRatio");
258        checkGlError("glGetUniformLocation uCRatio");
259        if (muMVPMatrixHandle == -1) {
260            throw new RuntimeException("Could not get attrib location for uCRatio");
261        }
262
263        /*
264         * Create our texture. This has to be done each time the
265         * surface is created.
266         */
267
268        int[] textures = new int[1];
269        GLES20.glGenTextures(1, textures, 0);
270
271        mTextureID = textures[0];
272        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
273        checkGlError("glBindTexture mTextureID");
274
275        // Can't do mipmapping with camera source
276        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
277                GLES20.GL_NEAREST);
278        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
279                GLES20.GL_LINEAR);
280        // Clamp to edge is the only option
281        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
282                GLES20.GL_CLAMP_TO_EDGE);
283        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
284                GLES20.GL_CLAMP_TO_EDGE);
285        checkGlError("glTexParameteri mTextureID");
286
287        /*
288         * Create the SurfaceTexture that will feed this textureID, and pass it to the camera
289         */
290
291        mSurface = new SurfaceTexture(mTextureID);
292        mSurface.setOnFrameAvailableListener(this);
293
294        Matrix.setLookAtM(mVMatrix, 0, 0, 0, 5f, 0f, 0f, 0f, 0f, 1.0f, 0.0f);
295
296        mLastTime = 0;
297
298        synchronized(this) {
299            updateSurface = false;
300        }
301    }
302
303    synchronized public void onFrameAvailable(SurfaceTexture surface) {
304        /* For simplicity, SurfaceTexture calls here when it has new
305         * data available.  Call may come in from some random thread,
306         * so let's be safe and use synchronize. No OpenGL calls can be done here.
307         */
308        updateSurface = true;
309        //Log.v(TAG, "onFrameAvailable " + surface.getTimestamp());
310    }
311
312    private void doPhysics(long timestamp) {
313        /*
314         * Move the camera surface around based on some simple spring physics with drag
315         */
316
317        if (mLastTime == 0)
318            mLastTime = timestamp;
319
320        float deltaT = (timestamp - mLastTime)/1000000000.f; // To seconds
321
322        float springStrength = 20.f;
323        float frictionCoeff = 10.f;
324        float mass = 10.f;
325        float gMultiplier = 4.f;
326        /* Only update physics every 30 ms */
327        if (deltaT > 0.030f) {
328            mLastTime = timestamp;
329
330            float[] totalForce = new float[3];
331            totalForce[0] = -mPos[0] * springStrength - mVel[0]*frictionCoeff + gMultiplier*mGForce[0]*mass;
332            totalForce[1] = -mPos[1] * springStrength - mVel[1]*frictionCoeff + gMultiplier*mGForce[1]*mass;
333            totalForce[2] = -mPos[2] * springStrength - mVel[2]*frictionCoeff + gMultiplier*mGForce[2]*mass;
334
335            float[] accel = new float[3];
336            accel[0] = totalForce[0]/mass;
337            accel[1] = totalForce[1]/mass;
338            accel[2] = totalForce[2]/mass;
339
340            /* Not a very accurate integrator */
341            mVel[0] = mVel[0] + accel[0]*deltaT;
342            mVel[1] = mVel[1] + accel[1]*deltaT;
343            mVel[2] = mVel[2] + accel[2]*deltaT;
344
345            mPos[0] = mPos[0] + mVel[0]*deltaT;
346            mPos[1] = mPos[1] + mVel[1]*deltaT;
347            mPos[2] = mPos[2] + mVel[2]*deltaT;
348
349            Matrix.setIdentityM(mMMatrix, 0);
350            Matrix.translateM(mMMatrix, 0, mPos[0], mPos[1], mPos[2]);
351        }
352
353    }
354
355    private int loadShader(int shaderType, String source) {
356        int shader = GLES20.glCreateShader(shaderType);
357        if (shader != 0) {
358            GLES20.glShaderSource(shader, source);
359            GLES20.glCompileShader(shader);
360            int[] compiled = new int[1];
361            GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
362            if (compiled[0] == 0) {
363                Log.e(TAG, "Could not compile shader " + shaderType + ":");
364                Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
365                GLES20.glDeleteShader(shader);
366                shader = 0;
367            }
368        }
369        return shader;
370    }
371
372    private int createProgram(String vertexSource, String fragmentSource) {
373        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
374        if (vertexShader == 0) {
375            return 0;
376        }
377        int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
378        if (pixelShader == 0) {
379            return 0;
380        }
381
382        int program = GLES20.glCreateProgram();
383        if (program != 0) {
384            GLES20.glAttachShader(program, vertexShader);
385            checkGlError("glAttachShader");
386            GLES20.glAttachShader(program, pixelShader);
387            checkGlError("glAttachShader");
388            GLES20.glLinkProgram(program);
389            int[] linkStatus = new int[1];
390            GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
391            if (linkStatus[0] != GLES20.GL_TRUE) {
392                Log.e(TAG, "Could not link program: ");
393                Log.e(TAG, GLES20.glGetProgramInfoLog(program));
394                GLES20.glDeleteProgram(program);
395                program = 0;
396            }
397        }
398        return program;
399    }
400
401    private void checkGlError(String op) {
402        int error;
403        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
404            Log.e(TAG, op + ": glError " + error);
405            throw new RuntimeException(op + ": glError " + error);
406        }
407    }
408
409    private static final int FLOAT_SIZE_BYTES = 4;
410    private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
411    private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
412    private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
413    private final float[] mTriangleVerticesData = {
414        // X, Y, Z, U, V
415        -1.0f, -1.0f, 0, 0.f, 0.f,
416        1.0f, -1.0f, 0, 1.f, 0.f,
417        -1.0f,  1.0f, 0, 0.f, 1.f,
418        1.0f,   1.0f, 0, 1.f, 1.f,
419    };
420
421    private FloatBuffer mTriangleVertices;
422
423    private final String mVertexShader =
424        "uniform mat4 uMVPMatrix;\n" +
425        "uniform mat4 uSTMatrix;\n" +
426        "uniform float uCRatio;\n" +
427        "attribute vec4 aPosition;\n" +
428        "attribute vec4 aTextureCoord;\n" +
429        "varying vec2 vTextureCoord;\n" +
430        "varying vec2 vTextureNormCoord;\n" +
431        "void main() {\n" +
432        "  vec4 scaledPos = aPosition;\n" +
433        "  scaledPos.x = scaledPos.x * uCRatio;\n" +
434        "  gl_Position = uMVPMatrix * scaledPos;\n" +
435        "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
436        "  vTextureNormCoord = aTextureCoord.xy;\n" +
437        "}\n";
438
439    private final String mFragmentShader =
440        "#extension GL_OES_EGL_image_external : require\n" +
441        "precision mediump float;\n" +
442        "varying vec2 vTextureCoord;\n" +
443        "varying vec2 vTextureNormCoord;\n" +
444        "uniform samplerExternalOES sTexture;\n" +
445        "void main() {\n" +
446        "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
447        "  gl_FragColor.a = 1.0-min(length(vTextureNormCoord-0.5)*2.0,1.0);\n" +
448        "}\n";
449
450    private float[] mMVPMatrix = new float[16];
451    private float[] mProjMatrix = new float[16];
452    private float[] mMMatrix = new float[16];
453    private float[] mVMatrix = new float[16];
454    private float[] mSTMatrix = new float[16];
455
456    private int mProgram;
457    private int mTextureID;
458    private int muMVPMatrixHandle;
459    private int muSTMatrixHandle;
460    private int muCRatioHandle;
461    private int maPositionHandle;
462    private int maTextureHandle;
463
464    private float mRatio = 1.0f;
465    private float mMediaPlayerRatio = 1.0f;
466    private float[] mVel = new float[3];
467    private float[] mPos = new float[3];
468    private float[] mGForce = new float[3];
469
470    private long mLastTime;
471
472    private SurfaceTexture mSurface;
473    private boolean updateSurface = false;
474
475    private Context mContext;
476    private static final String TAG = "MyRenderer";
477
478    // Magic key
479    private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
480
481    public SurfaceTexture getSurfaceTexture()
482    {
483        return mSurface;
484    }
485
486}
487