1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package android.hardware.camera2.legacy;
17
18import android.graphics.ImageFormat;
19import android.graphics.RectF;
20import android.graphics.SurfaceTexture;
21import android.hardware.camera2.CameraCharacteristics;
22import android.os.Environment;
23import android.opengl.EGL14;
24import android.opengl.EGLConfig;
25import android.opengl.EGLContext;
26import android.opengl.EGLDisplay;
27import android.opengl.EGLSurface;
28import android.opengl.GLES11Ext;
29import android.opengl.GLES20;
30import android.opengl.Matrix;
31import android.text.format.Time;
32import android.util.Log;
33import android.util.Pair;
34import android.util.Size;
35import android.view.Surface;
36import android.os.SystemProperties;
37
38import java.io.File;
39import java.nio.ByteBuffer;
40import java.nio.ByteOrder;
41import java.nio.FloatBuffer;
42import java.util.ArrayList;
43import java.util.Collection;
44import java.util.List;
45
46/**
47 * A renderer class that manages the GL state, and can draw a frame into a set of output
48 * {@link Surface}s.
49 */
50public class SurfaceTextureRenderer {
51    private static final String TAG = SurfaceTextureRenderer.class.getSimpleName();
52    private static final boolean DEBUG = false;
53    private static final int EGL_RECORDABLE_ANDROID = 0x3142; // from EGL/eglext.h
54    private static final int GL_MATRIX_SIZE = 16;
55    private static final int VERTEX_POS_SIZE = 3;
56    private static final int VERTEX_UV_SIZE = 2;
57    private static final int EGL_COLOR_BITLENGTH = 8;
58    private static final int GLES_VERSION = 2;
59    private static final int PBUFFER_PIXEL_BYTES = 4;
60
61    private static final int FLIP_TYPE_NONE = 0;
62    private static final int FLIP_TYPE_HORIZONTAL = 1;
63    private static final int FLIP_TYPE_VERTICAL = 2;
64    private static final int FLIP_TYPE_BOTH = FLIP_TYPE_HORIZONTAL | FLIP_TYPE_VERTICAL;
65
66    private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
67    private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
68    private EGLConfig mConfigs;
69
70    private class EGLSurfaceHolder {
71        Surface surface;
72        EGLSurface eglSurface;
73        int width;
74        int height;
75    }
76
77    private List<EGLSurfaceHolder> mSurfaces = new ArrayList<EGLSurfaceHolder>();
78    private List<EGLSurfaceHolder> mConversionSurfaces = new ArrayList<EGLSurfaceHolder>();
79
80    private ByteBuffer mPBufferPixels;
81
82    // Hold this to avoid GC
83    private volatile SurfaceTexture mSurfaceTexture;
84
85    private static final int FLOAT_SIZE_BYTES = 4;
86    private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
87    private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
88    private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
89
90    // Sampling is mirrored across the horizontal axis
91    private static final float[] sHorizontalFlipTriangleVertices = {
92            // X, Y, Z, U, V
93            -1.0f, -1.0f, 0, 1.f, 0.f,
94            1.0f, -1.0f, 0, 0.f, 0.f,
95            -1.0f,  1.0f, 0, 1.f, 1.f,
96            1.0f,  1.0f, 0, 0.f, 1.f,
97    };
98
99    // Sampling is mirrored across the vertical axis
100    private static final float[] sVerticalFlipTriangleVertices = {
101            // X, Y, Z, U, V
102            -1.0f, -1.0f, 0, 0.f, 1.f,
103            1.0f, -1.0f, 0, 1.f, 1.f,
104            -1.0f,  1.0f, 0, 0.f, 0.f,
105            1.0f,  1.0f, 0, 1.f, 0.f,
106    };
107
108    // Sampling is mirrored across the both axes
109    private static final float[] sBothFlipTriangleVertices = {
110            // X, Y, Z, U, V
111            -1.0f, -1.0f, 0, 1.f, 1.f,
112            1.0f, -1.0f, 0, 0.f, 1.f,
113            -1.0f,  1.0f, 0, 1.f, 0.f,
114            1.0f,  1.0f, 0, 0.f, 0.f,
115    };
116
117    // Sampling is 1:1 for a straight copy for the back camera
118    private static final float[] sRegularTriangleVertices = {
119            // X, Y, Z, U, V
120            -1.0f, -1.0f, 0, 0.f, 0.f,
121            1.0f, -1.0f, 0, 1.f, 0.f,
122            -1.0f,  1.0f, 0, 0.f, 1.f,
123            1.0f,  1.0f, 0, 1.f, 1.f,
124    };
125
126    private FloatBuffer mRegularTriangleVertices;
127    private FloatBuffer mHorizontalFlipTriangleVertices;
128    private FloatBuffer mVerticalFlipTriangleVertices;
129    private FloatBuffer mBothFlipTriangleVertices;
130    private final int mFacing;
131
132    /**
133     * As used in this file, this vertex shader maps a unit square to the view, and
134     * tells the fragment shader to interpolate over it.  Each surface pixel position
135     * is mapped to a 2D homogeneous texture coordinate of the form (s, t, 0, 1) with
136     * s and t in the inclusive range [0, 1], and the matrix from
137     * {@link SurfaceTexture#getTransformMatrix(float[])} is used to map this
138     * coordinate to a texture location.
139     */
140    private static final String VERTEX_SHADER =
141            "uniform mat4 uMVPMatrix;\n" +
142            "uniform mat4 uSTMatrix;\n" +
143            "attribute vec4 aPosition;\n" +
144            "attribute vec4 aTextureCoord;\n" +
145            "varying vec2 vTextureCoord;\n" +
146            "void main() {\n" +
147            "  gl_Position = uMVPMatrix * aPosition;\n" +
148            "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
149            "}\n";
150
151    /**
152     * This fragment shader simply draws the color in the 2D texture at
153     * the location from the {@code VERTEX_SHADER}.
154     */
155    private static final String FRAGMENT_SHADER =
156            "#extension GL_OES_EGL_image_external : require\n" +
157            "precision mediump float;\n" +
158            "varying vec2 vTextureCoord;\n" +
159            "uniform samplerExternalOES sTexture;\n" +
160            "void main() {\n" +
161            "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
162            "}\n";
163
164    private float[] mMVPMatrix = new float[GL_MATRIX_SIZE];
165    private float[] mSTMatrix = new float[GL_MATRIX_SIZE];
166
167    private int mProgram;
168    private int mTextureID = 0;
169    private int muMVPMatrixHandle;
170    private int muSTMatrixHandle;
171    private int maPositionHandle;
172    private int maTextureHandle;
173
174    private PerfMeasurement mPerfMeasurer = null;
175    private static final String LEGACY_PERF_PROPERTY = "persist.camera.legacy_perf";
176
177    public SurfaceTextureRenderer(int facing) {
178        mFacing = facing;
179
180        mRegularTriangleVertices = ByteBuffer.allocateDirect(sRegularTriangleVertices.length *
181                FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
182        mRegularTriangleVertices.put(sRegularTriangleVertices).position(0);
183
184        mHorizontalFlipTriangleVertices = ByteBuffer.allocateDirect(
185                sHorizontalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
186                order(ByteOrder.nativeOrder()).asFloatBuffer();
187        mHorizontalFlipTriangleVertices.put(sHorizontalFlipTriangleVertices).position(0);
188
189        mVerticalFlipTriangleVertices = ByteBuffer.allocateDirect(
190                sVerticalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
191                order(ByteOrder.nativeOrder()).asFloatBuffer();
192        mVerticalFlipTriangleVertices.put(sVerticalFlipTriangleVertices).position(0);
193
194        mBothFlipTriangleVertices = ByteBuffer.allocateDirect(
195                sBothFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
196                order(ByteOrder.nativeOrder()).asFloatBuffer();
197        mBothFlipTriangleVertices.put(sBothFlipTriangleVertices).position(0);
198
199        Matrix.setIdentityM(mSTMatrix, 0);
200    }
201
202    private int loadShader(int shaderType, String source) {
203        int shader = GLES20.glCreateShader(shaderType);
204        checkGlError("glCreateShader type=" + shaderType);
205        GLES20.glShaderSource(shader, source);
206        GLES20.glCompileShader(shader);
207        int[] compiled = new int[1];
208        GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
209        if (compiled[0] == 0) {
210            Log.e(TAG, "Could not compile shader " + shaderType + ":");
211            Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
212            GLES20.glDeleteShader(shader);
213            // TODO: handle this more gracefully
214            throw new IllegalStateException("Could not compile shader " + shaderType);
215        }
216        return shader;
217    }
218
219    private int createProgram(String vertexSource, String fragmentSource) {
220        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
221        if (vertexShader == 0) {
222            return 0;
223        }
224        int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
225        if (pixelShader == 0) {
226            return 0;
227        }
228
229        int program = GLES20.glCreateProgram();
230        checkGlError("glCreateProgram");
231        if (program == 0) {
232            Log.e(TAG, "Could not create program");
233        }
234        GLES20.glAttachShader(program, vertexShader);
235        checkGlError("glAttachShader");
236        GLES20.glAttachShader(program, pixelShader);
237        checkGlError("glAttachShader");
238        GLES20.glLinkProgram(program);
239        int[] linkStatus = new int[1];
240        GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
241        if (linkStatus[0] != GLES20.GL_TRUE) {
242            Log.e(TAG, "Could not link program: ");
243            Log.e(TAG, GLES20.glGetProgramInfoLog(program));
244            GLES20.glDeleteProgram(program);
245            // TODO: handle this more gracefully
246            throw new IllegalStateException("Could not link program");
247        }
248        return program;
249    }
250
251    private void drawFrame(SurfaceTexture st, int width, int height, int flipType)
252            throws LegacyExceptionUtils.BufferQueueAbandonedException {
253        checkGlError("onDrawFrame start");
254        st.getTransformMatrix(mSTMatrix);
255
256        Matrix.setIdentityM(mMVPMatrix, /*smOffset*/0);
257
258        // Find intermediate buffer dimensions
259        Size dimens;
260        try {
261            dimens = LegacyCameraDevice.getTextureSize(st);
262        } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
263            // Should never hit this.
264            throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e);
265        }
266        float texWidth = dimens.getWidth();
267        float texHeight = dimens.getHeight();
268
269        if (texWidth <= 0 || texHeight <= 0) {
270            throw new IllegalStateException("Illegal intermediate texture with dimension of 0");
271        }
272
273        // Letterbox or pillar-box output dimensions into intermediate dimensions.
274        RectF intermediate = new RectF(/*left*/0, /*top*/0, /*right*/texWidth, /*bottom*/texHeight);
275        RectF output = new RectF(/*left*/0, /*top*/0, /*right*/width, /*bottom*/height);
276        android.graphics.Matrix boxingXform = new android.graphics.Matrix();
277        boxingXform.setRectToRect(output, intermediate, android.graphics.Matrix.ScaleToFit.CENTER);
278        boxingXform.mapRect(output);
279
280        // Find scaling factor from pillar-boxed/letter-boxed output dimensions to intermediate
281        // buffer dimensions.
282        float scaleX = intermediate.width() / output.width();
283        float scaleY = intermediate.height() / output.height();
284
285        // Intermediate texture is implicitly scaled to 'fill' the output dimensions in clip space
286        // coordinates in the shader.  To avoid stretching, we need to scale the larger dimension
287        // of the intermediate buffer so that the output buffer is actually letter-boxed
288        // or pillar-boxed into the intermediate buffer after clipping.
289        Matrix.scaleM(mMVPMatrix, /*offset*/0, /*x*/scaleX, /*y*/scaleY, /*z*/1);
290
291        if (DEBUG) {
292            Log.d(TAG, "Scaling factors (S_x = " + scaleX + ",S_y = " + scaleY + ") used for " +
293                    width + "x" + height + " surface, intermediate buffer size is " + texWidth +
294                    "x" + texHeight);
295        }
296
297        // Set viewport to be output buffer dimensions
298        GLES20.glViewport(0, 0, width, height);
299
300        if (DEBUG) {
301            GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
302            GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
303        }
304
305        GLES20.glUseProgram(mProgram);
306        checkGlError("glUseProgram");
307
308        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
309        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
310
311        FloatBuffer triangleVertices;
312        switch(flipType) {
313            case FLIP_TYPE_HORIZONTAL:
314                triangleVertices = mHorizontalFlipTriangleVertices;
315                break;
316            case FLIP_TYPE_VERTICAL:
317                triangleVertices = mVerticalFlipTriangleVertices;
318                break;
319            case FLIP_TYPE_BOTH:
320                triangleVertices = mBothFlipTriangleVertices;
321                break;
322            default:
323                triangleVertices = mRegularTriangleVertices;
324                break;
325        }
326
327        triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
328        GLES20.glVertexAttribPointer(maPositionHandle, VERTEX_POS_SIZE, GLES20.GL_FLOAT,
329                /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
330        checkGlError("glVertexAttribPointer maPosition");
331        GLES20.glEnableVertexAttribArray(maPositionHandle);
332        checkGlError("glEnableVertexAttribArray maPositionHandle");
333
334        triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
335        GLES20.glVertexAttribPointer(maTextureHandle, VERTEX_UV_SIZE, GLES20.GL_FLOAT,
336                /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
337        checkGlError("glVertexAttribPointer maTextureHandle");
338        GLES20.glEnableVertexAttribArray(maTextureHandle);
339        checkGlError("glEnableVertexAttribArray maTextureHandle");
340
341        GLES20.glUniformMatrix4fv(muMVPMatrixHandle, /*count*/ 1, /*transpose*/ false, mMVPMatrix,
342                /*offset*/ 0);
343        GLES20.glUniformMatrix4fv(muSTMatrixHandle, /*count*/ 1, /*transpose*/ false, mSTMatrix,
344                /*offset*/ 0);
345
346        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*offset*/ 0, /*count*/ 4);
347        checkGlDrawError("glDrawArrays");
348    }
349
350    /**
351     * Initializes GL state.  Call this after the EGL surface has been created and made current.
352     */
353    private void initializeGLState() {
354        mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
355        if (mProgram == 0) {
356            throw new IllegalStateException("failed creating program");
357        }
358        maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
359        checkGlError("glGetAttribLocation aPosition");
360        if (maPositionHandle == -1) {
361            throw new IllegalStateException("Could not get attrib location for aPosition");
362        }
363        maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
364        checkGlError("glGetAttribLocation aTextureCoord");
365        if (maTextureHandle == -1) {
366            throw new IllegalStateException("Could not get attrib location for aTextureCoord");
367        }
368
369        muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
370        checkGlError("glGetUniformLocation uMVPMatrix");
371        if (muMVPMatrixHandle == -1) {
372            throw new IllegalStateException("Could not get attrib location for uMVPMatrix");
373        }
374
375        muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
376        checkGlError("glGetUniformLocation uSTMatrix");
377        if (muSTMatrixHandle == -1) {
378            throw new IllegalStateException("Could not get attrib location for uSTMatrix");
379        }
380
381        int[] textures = new int[1];
382        GLES20.glGenTextures(/*n*/ 1, textures, /*offset*/ 0);
383
384        mTextureID = textures[0];
385        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
386        checkGlError("glBindTexture mTextureID");
387
388        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
389                GLES20.GL_NEAREST);
390        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
391                GLES20.GL_LINEAR);
392        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
393                GLES20.GL_CLAMP_TO_EDGE);
394        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
395                GLES20.GL_CLAMP_TO_EDGE);
396        checkGlError("glTexParameter");
397    }
398
399    private int getTextureId() {
400        return mTextureID;
401    }
402
403    private void clearState() {
404        mSurfaces.clear();
405        for (EGLSurfaceHolder holder : mConversionSurfaces) {
406            try {
407                LegacyCameraDevice.disconnectSurface(holder.surface);
408            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
409                Log.w(TAG, "Surface abandoned, skipping...", e);
410            }
411        }
412        mConversionSurfaces.clear();
413        mPBufferPixels = null;
414        if (mSurfaceTexture != null) {
415            mSurfaceTexture.release();
416        }
417        mSurfaceTexture = null;
418    }
419
420    private void configureEGLContext() {
421        mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
422        if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
423            throw new IllegalStateException("No EGL14 display");
424        }
425        int[] version = new int[2];
426        if (!EGL14.eglInitialize(mEGLDisplay, version, /*offset*/ 0, version, /*offset*/ 1)) {
427            throw new IllegalStateException("Cannot initialize EGL14");
428        }
429
430        int[] attribList = {
431                EGL14.EGL_RED_SIZE, EGL_COLOR_BITLENGTH,
432                EGL14.EGL_GREEN_SIZE, EGL_COLOR_BITLENGTH,
433                EGL14.EGL_BLUE_SIZE, EGL_COLOR_BITLENGTH,
434                EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
435                EGL_RECORDABLE_ANDROID, 1,
436                EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT | EGL14.EGL_WINDOW_BIT,
437                EGL14.EGL_NONE
438        };
439        EGLConfig[] configs = new EGLConfig[1];
440        int[] numConfigs = new int[1];
441        EGL14.eglChooseConfig(mEGLDisplay, attribList, /*offset*/ 0, configs, /*offset*/ 0,
442                configs.length, numConfigs, /*offset*/ 0);
443        checkEglError("eglCreateContext RGB888+recordable ES2");
444        mConfigs = configs[0];
445        int[] attrib_list = {
446                EGL14.EGL_CONTEXT_CLIENT_VERSION, GLES_VERSION,
447                EGL14.EGL_NONE
448        };
449        mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
450                attrib_list, /*offset*/ 0);
451        checkEglError("eglCreateContext");
452        if(mEGLContext == EGL14.EGL_NO_CONTEXT) {
453            throw new IllegalStateException("No EGLContext could be made");
454        }
455    }
456
457    private void configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces) {
458        if (surfaces == null || surfaces.size() == 0) {
459            throw new IllegalStateException("No Surfaces were provided to draw to");
460        }
461        int[] surfaceAttribs = {
462                EGL14.EGL_NONE
463        };
464        for (EGLSurfaceHolder holder : surfaces) {
465            holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
466                    holder.surface, surfaceAttribs, /*offset*/ 0);
467            checkEglError("eglCreateWindowSurface");
468        }
469    }
470
471    private void configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces) {
472        if (surfaces == null || surfaces.size() == 0) {
473            throw new IllegalStateException("No Surfaces were provided to draw to");
474        }
475
476        int maxLength = 0;
477        for (EGLSurfaceHolder holder : surfaces) {
478            int length = holder.width * holder.height;
479            // Find max surface size, ensure PBuffer can hold this many pixels
480            maxLength = (length > maxLength) ? length : maxLength;
481            int[] surfaceAttribs = {
482                    EGL14.EGL_WIDTH, holder.width,
483                    EGL14.EGL_HEIGHT, holder.height,
484                    EGL14.EGL_NONE
485            };
486            holder.eglSurface =
487                    EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
488            checkEglError("eglCreatePbufferSurface");
489        }
490        mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES)
491                .order(ByteOrder.nativeOrder());
492    }
493
494    private void releaseEGLContext() {
495        if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
496            EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
497                    EGL14.EGL_NO_CONTEXT);
498            dumpGlTiming();
499            if (mSurfaces != null) {
500                for (EGLSurfaceHolder holder : mSurfaces) {
501                    if (holder.eglSurface != null) {
502                        EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
503                    }
504                }
505            }
506            if (mConversionSurfaces != null) {
507                for (EGLSurfaceHolder holder : mConversionSurfaces) {
508                    if (holder.eglSurface != null) {
509                        EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
510                    }
511                }
512            }
513            EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
514            EGL14.eglReleaseThread();
515            EGL14.eglTerminate(mEGLDisplay);
516        }
517
518        mConfigs = null;
519        mEGLDisplay = EGL14.EGL_NO_DISPLAY;
520        mEGLContext = EGL14.EGL_NO_CONTEXT;
521        clearState();
522    }
523
524    private void makeCurrent(EGLSurface surface) {
525        EGL14.eglMakeCurrent(mEGLDisplay, surface, surface, mEGLContext);
526        checkEglError("makeCurrent");
527    }
528
529    private boolean swapBuffers(EGLSurface surface)
530            throws LegacyExceptionUtils.BufferQueueAbandonedException {
531        boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface);
532        int error = EGL14.eglGetError();
533        if (error == EGL14.EGL_BAD_SURFACE) {
534            throw new LegacyExceptionUtils.BufferQueueAbandonedException();
535        } else if (error != EGL14.EGL_SUCCESS) {
536            throw new IllegalStateException("swapBuffers: EGL error: 0x" +
537                    Integer.toHexString(error));
538        }
539        return result;
540    }
541
542    private void checkEglError(String msg) {
543        int error;
544        if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
545            throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error));
546        }
547    }
548
549    private void checkGlError(String msg) {
550        int error;
551        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
552            throw new IllegalStateException(
553                    msg + ": GLES20 error: 0x" + Integer.toHexString(error));
554        }
555    }
556
557    private void checkGlDrawError(String msg)
558            throws LegacyExceptionUtils.BufferQueueAbandonedException {
559        int error;
560        boolean surfaceAbandoned = false;
561        boolean glError = false;
562        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
563            if (error == GLES20.GL_OUT_OF_MEMORY) {
564                surfaceAbandoned = true;
565            } else {
566                glError = true;
567            }
568        }
569        if (glError) {
570            throw new IllegalStateException(
571                    msg + ": GLES20 error: 0x" + Integer.toHexString(error));
572        }
573        if (surfaceAbandoned) {
574            throw new LegacyExceptionUtils.BufferQueueAbandonedException();
575        }
576    }
577
578    /**
579     * Save a measurement dump to disk, in
580     * {@code /sdcard/CameraLegacy/durations_<time>_<width1>x<height1>_...txt}
581     */
582    private void dumpGlTiming() {
583        if (mPerfMeasurer == null) return;
584
585        File legacyStorageDir = new File(Environment.getExternalStorageDirectory(), "CameraLegacy");
586        if (!legacyStorageDir.exists()){
587            if (!legacyStorageDir.mkdirs()){
588                Log.e(TAG, "Failed to create directory for data dump");
589                return;
590            }
591        }
592
593        StringBuilder path = new StringBuilder(legacyStorageDir.getPath());
594        path.append(File.separator);
595        path.append("durations_");
596
597        Time now = new Time();
598        now.setToNow();
599        path.append(now.format2445());
600        path.append("_S");
601        for (EGLSurfaceHolder surface : mSurfaces) {
602            path.append(String.format("_%d_%d", surface.width, surface.height));
603        }
604        path.append("_C");
605        for (EGLSurfaceHolder surface : mConversionSurfaces) {
606            path.append(String.format("_%d_%d", surface.width, surface.height));
607        }
608        path.append(".txt");
609        mPerfMeasurer.dumpPerformanceData(path.toString());
610    }
611
612    private void setupGlTiming() {
613        if (PerfMeasurement.isGlTimingSupported()) {
614            Log.d(TAG, "Enabling GL performance measurement");
615            mPerfMeasurer = new PerfMeasurement();
616        } else {
617            Log.d(TAG, "GL performance measurement not supported on this device");
618            mPerfMeasurer = null;
619        }
620    }
621
622    private void beginGlTiming() {
623        if (mPerfMeasurer == null) return;
624        mPerfMeasurer.startTimer();
625    }
626
627    private void addGlTimestamp(long timestamp) {
628        if (mPerfMeasurer == null) return;
629        mPerfMeasurer.addTimestamp(timestamp);
630    }
631
632    private void endGlTiming() {
633        if (mPerfMeasurer == null) return;
634        mPerfMeasurer.stopTimer();
635    }
636
637    /**
638     * Return the surface texture to draw to - this is the texture use to when producing output
639     * surface buffers.
640     *
641     * @return a {@link SurfaceTexture}.
642     */
643    public SurfaceTexture getSurfaceTexture() {
644        return mSurfaceTexture;
645    }
646
647    /**
648     * Set a collection of output {@link Surface}s that can be drawn to.
649     *
650     * @param surfaces a {@link Collection} of surfaces.
651     */
652    public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) {
653        releaseEGLContext();
654
655        if (surfaces == null || surfaces.size() == 0) {
656            Log.w(TAG, "No output surfaces configured for GL drawing.");
657            return;
658        }
659
660        for (Pair<Surface, Size> p : surfaces) {
661            Surface s = p.first;
662            Size surfaceSize = p.second;
663            // If pixel conversions aren't handled by egl, use a pbuffer
664            try {
665                EGLSurfaceHolder holder = new EGLSurfaceHolder();
666                holder.surface = s;
667                holder.width = surfaceSize.getWidth();
668                holder.height = surfaceSize.getHeight();
669                if (LegacyCameraDevice.needsConversion(s)) {
670                    mConversionSurfaces.add(holder);
671                    // LegacyCameraDevice is the producer of surfaces if it's not handled by EGL,
672                    // so LegacyCameraDevice needs to connect to the surfaces.
673                    LegacyCameraDevice.connectSurface(s);
674                } else {
675                    mSurfaces.add(holder);
676                }
677            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
678                Log.w(TAG, "Surface abandoned, skipping configuration... ", e);
679            }
680        }
681
682        // Set up egl display
683        configureEGLContext();
684
685        // Set up regular egl surfaces if needed
686        if (mSurfaces.size() > 0) {
687            configureEGLOutputSurfaces(mSurfaces);
688        }
689
690        // Set up pbuffer surface if needed
691        if (mConversionSurfaces.size() > 0) {
692            configureEGLPbufferSurfaces(mConversionSurfaces);
693        }
694        makeCurrent((mSurfaces.size() > 0) ? mSurfaces.get(0).eglSurface :
695                mConversionSurfaces.get(0).eglSurface);
696        initializeGLState();
697        mSurfaceTexture = new SurfaceTexture(getTextureId());
698
699        // Set up performance tracking if enabled
700        if (SystemProperties.getBoolean(LEGACY_PERF_PROPERTY, false)) {
701            setupGlTiming();
702        }
703    }
704
705    /**
706     * Draw the current buffer in the {@link SurfaceTexture} returned from
707     * {@link #getSurfaceTexture()} into the set of target {@link Surface}s
708     * in the next request from the given {@link CaptureCollector}, or drop
709     * the frame if none is available.
710     *
711     * <p>
712     * Any {@link Surface}s targeted must be a subset of the {@link Surface}s
713     * set in the last {@link #configureSurfaces(java.util.Collection)} call.
714     * </p>
715     *
716     * @param targetCollector the surfaces to draw to.
717     */
718    public void drawIntoSurfaces(CaptureCollector targetCollector) {
719        if ((mSurfaces == null || mSurfaces.size() == 0)
720                && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) {
721            return;
722        }
723
724        boolean doTiming = targetCollector.hasPendingPreviewCaptures();
725        checkGlError("before updateTexImage");
726
727        if (doTiming) {
728            beginGlTiming();
729        }
730
731        mSurfaceTexture.updateTexImage();
732
733        long timestamp = mSurfaceTexture.getTimestamp();
734
735        Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp);
736
737        // No preview request queued, drop frame.
738        if (captureHolder == null) {
739            if (DEBUG) {
740                Log.d(TAG, "Dropping preview frame.");
741            }
742            if (doTiming) {
743                endGlTiming();
744            }
745            return;
746        }
747
748        RequestHolder request = captureHolder.first;
749
750        Collection<Surface> targetSurfaces = request.getHolderTargets();
751        if (doTiming) {
752            addGlTimestamp(timestamp);
753        }
754
755        List<Long> targetSurfaceIds = new ArrayList();
756        try {
757            targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
758        } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
759            Log.w(TAG, "Surface abandoned, dropping frame. ", e);
760            request.setOutputAbandoned();
761        }
762
763        for (EGLSurfaceHolder holder : mSurfaces) {
764            if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
765                try{
766                    LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
767                            holder.height);
768                    makeCurrent(holder.eglSurface);
769
770                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
771                    drawFrame(mSurfaceTexture, holder.width, holder.height,
772                            (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
773                                    FLIP_TYPE_HORIZONTAL : FLIP_TYPE_NONE);
774                    swapBuffers(holder.eglSurface);
775                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
776                    Log.w(TAG, "Surface abandoned, dropping frame. ", e);
777                    request.setOutputAbandoned();
778                }
779            }
780        }
781        for (EGLSurfaceHolder holder : mConversionSurfaces) {
782            if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
783                makeCurrent(holder.eglSurface);
784                // glReadPixels reads from the bottom of the buffer, so add an extra vertical flip
785                try {
786                    drawFrame(mSurfaceTexture, holder.width, holder.height,
787                            (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
788                                    FLIP_TYPE_BOTH : FLIP_TYPE_VERTICAL);
789                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
790                    // Should never hit this.
791                    throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e);
792                }
793                mPBufferPixels.clear();
794                GLES20.glReadPixels(/*x*/ 0, /*y*/ 0, holder.width, holder.height,
795                        GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPBufferPixels);
796                checkGlError("glReadPixels");
797
798                try {
799                    int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
800                    LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
801                            holder.height);
802                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
803                    LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
804                            holder.width, holder.height, format);
805                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
806                    Log.w(TAG, "Surface abandoned, dropping frame. ", e);
807                    request.setOutputAbandoned();
808                }
809            }
810        }
811        targetCollector.previewProduced();
812
813        if (doTiming) {
814            endGlTiming();
815        }
816    }
817
818    /**
819     * Clean up the current GL context.
820     */
821    public void cleanupEGLContext() {
822        releaseEGLContext();
823    }
824
825    /**
826     * Drop all current GL operations on the floor.
827     */
828    public void flush() {
829        // TODO: implement flush
830        Log.e(TAG, "Flush not yet implemented.");
831    }
832}
833