1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package android.hardware.camera2.legacy;
17
18import android.graphics.ImageFormat;
19import android.graphics.RectF;
20import android.graphics.SurfaceTexture;
21import android.hardware.camera2.CameraCharacteristics;
22import android.os.Environment;
23import android.opengl.EGL14;
24import android.opengl.EGLConfig;
25import android.opengl.EGLContext;
26import android.opengl.EGLDisplay;
27import android.opengl.EGLSurface;
28import android.opengl.GLES11Ext;
29import android.opengl.GLES20;
30import android.opengl.Matrix;
31import android.text.format.Time;
32import android.util.Log;
33import android.util.Pair;
34import android.util.Size;
35import android.view.Surface;
36import android.os.SystemProperties;
37
38import java.io.File;
39import java.nio.ByteBuffer;
40import java.nio.ByteOrder;
41import java.nio.FloatBuffer;
42import java.util.ArrayList;
43import java.util.Collection;
44import java.util.List;
45
46/**
47 * A renderer class that manages the GL state, and can draw a frame into a set of output
48 * {@link Surface}s.
49 */
50public class SurfaceTextureRenderer {
51    private static final String TAG = SurfaceTextureRenderer.class.getSimpleName();
52    private static final boolean DEBUG = false;
53    private static final int EGL_RECORDABLE_ANDROID = 0x3142; // from EGL/eglext.h
54    private static final int GL_MATRIX_SIZE = 16;
55    private static final int VERTEX_POS_SIZE = 3;
56    private static final int VERTEX_UV_SIZE = 2;
57    private static final int EGL_COLOR_BITLENGTH = 8;
58    private static final int GLES_VERSION = 2;
59    private static final int PBUFFER_PIXEL_BYTES = 4;
60
61    private static final int FLIP_TYPE_NONE = 0;
62    private static final int FLIP_TYPE_HORIZONTAL = 1;
63    private static final int FLIP_TYPE_VERTICAL = 2;
64    private static final int FLIP_TYPE_BOTH = FLIP_TYPE_HORIZONTAL | FLIP_TYPE_VERTICAL;
65
66    private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
67    private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
68    private EGLConfig mConfigs;
69
70    private class EGLSurfaceHolder {
71        Surface surface;
72        EGLSurface eglSurface;
73        int width;
74        int height;
75    }
76
77    private List<EGLSurfaceHolder> mSurfaces = new ArrayList<EGLSurfaceHolder>();
78    private List<EGLSurfaceHolder> mConversionSurfaces = new ArrayList<EGLSurfaceHolder>();
79
80    private ByteBuffer mPBufferPixels;
81
82    // Hold this to avoid GC
83    private volatile SurfaceTexture mSurfaceTexture;
84
85    private static final int FLOAT_SIZE_BYTES = 4;
86    private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
87    private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
88    private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
89
90    // Sampling is mirrored across the horizontal axis
91    private static final float[] sHorizontalFlipTriangleVertices = {
92            // X, Y, Z, U, V
93            -1.0f, -1.0f, 0, 1.f, 0.f,
94            1.0f, -1.0f, 0, 0.f, 0.f,
95            -1.0f,  1.0f, 0, 1.f, 1.f,
96            1.0f,  1.0f, 0, 0.f, 1.f,
97    };
98
99    // Sampling is mirrored across the vertical axis
100    private static final float[] sVerticalFlipTriangleVertices = {
101            // X, Y, Z, U, V
102            -1.0f, -1.0f, 0, 0.f, 1.f,
103            1.0f, -1.0f, 0, 1.f, 1.f,
104            -1.0f,  1.0f, 0, 0.f, 0.f,
105            1.0f,  1.0f, 0, 1.f, 0.f,
106    };
107
108    // Sampling is mirrored across the both axes
109    private static final float[] sBothFlipTriangleVertices = {
110            // X, Y, Z, U, V
111            -1.0f, -1.0f, 0, 1.f, 1.f,
112            1.0f, -1.0f, 0, 0.f, 1.f,
113            -1.0f,  1.0f, 0, 1.f, 0.f,
114            1.0f,  1.0f, 0, 0.f, 0.f,
115    };
116
117    // Sampling is 1:1 for a straight copy for the back camera
118    private static final float[] sRegularTriangleVertices = {
119            // X, Y, Z, U, V
120            -1.0f, -1.0f, 0, 0.f, 0.f,
121            1.0f, -1.0f, 0, 1.f, 0.f,
122            -1.0f,  1.0f, 0, 0.f, 1.f,
123            1.0f,  1.0f, 0, 1.f, 1.f,
124    };
125
126    private FloatBuffer mRegularTriangleVertices;
127    private FloatBuffer mHorizontalFlipTriangleVertices;
128    private FloatBuffer mVerticalFlipTriangleVertices;
129    private FloatBuffer mBothFlipTriangleVertices;
130    private final int mFacing;
131
132    /**
133     * As used in this file, this vertex shader maps a unit square to the view, and
134     * tells the fragment shader to interpolate over it.  Each surface pixel position
135     * is mapped to a 2D homogeneous texture coordinate of the form (s, t, 0, 1) with
136     * s and t in the inclusive range [0, 1], and the matrix from
137     * {@link SurfaceTexture#getTransformMatrix(float[])} is used to map this
138     * coordinate to a texture location.
139     */
140    private static final String VERTEX_SHADER =
141            "uniform mat4 uMVPMatrix;\n" +
142            "uniform mat4 uSTMatrix;\n" +
143            "attribute vec4 aPosition;\n" +
144            "attribute vec4 aTextureCoord;\n" +
145            "varying vec2 vTextureCoord;\n" +
146            "void main() {\n" +
147            "  gl_Position = uMVPMatrix * aPosition;\n" +
148            "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
149            "}\n";
150
151    /**
152     * This fragment shader simply draws the color in the 2D texture at
153     * the location from the {@code VERTEX_SHADER}.
154     */
155    private static final String FRAGMENT_SHADER =
156            "#extension GL_OES_EGL_image_external : require\n" +
157            "precision mediump float;\n" +
158            "varying vec2 vTextureCoord;\n" +
159            "uniform samplerExternalOES sTexture;\n" +
160            "void main() {\n" +
161            "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
162            "}\n";
163
164    private float[] mMVPMatrix = new float[GL_MATRIX_SIZE];
165    private float[] mSTMatrix = new float[GL_MATRIX_SIZE];
166
167    private int mProgram;
168    private int mTextureID = 0;
169    private int muMVPMatrixHandle;
170    private int muSTMatrixHandle;
171    private int maPositionHandle;
172    private int maTextureHandle;
173
174    private PerfMeasurement mPerfMeasurer = null;
175    private static final String LEGACY_PERF_PROPERTY = "persist.camera.legacy_perf";
176
177    public SurfaceTextureRenderer(int facing) {
178        mFacing = facing;
179
180        mRegularTriangleVertices = ByteBuffer.allocateDirect(sRegularTriangleVertices.length *
181                FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
182        mRegularTriangleVertices.put(sRegularTriangleVertices).position(0);
183
184        mHorizontalFlipTriangleVertices = ByteBuffer.allocateDirect(
185                sHorizontalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
186                order(ByteOrder.nativeOrder()).asFloatBuffer();
187        mHorizontalFlipTriangleVertices.put(sHorizontalFlipTriangleVertices).position(0);
188
189        mVerticalFlipTriangleVertices = ByteBuffer.allocateDirect(
190                sVerticalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
191                order(ByteOrder.nativeOrder()).asFloatBuffer();
192        mVerticalFlipTriangleVertices.put(sVerticalFlipTriangleVertices).position(0);
193
194        mBothFlipTriangleVertices = ByteBuffer.allocateDirect(
195                sBothFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
196                order(ByteOrder.nativeOrder()).asFloatBuffer();
197        mBothFlipTriangleVertices.put(sBothFlipTriangleVertices).position(0);
198
199        Matrix.setIdentityM(mSTMatrix, 0);
200    }
201
202    private int loadShader(int shaderType, String source) {
203        int shader = GLES20.glCreateShader(shaderType);
204        checkGlError("glCreateShader type=" + shaderType);
205        GLES20.glShaderSource(shader, source);
206        GLES20.glCompileShader(shader);
207        int[] compiled = new int[1];
208        GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
209        if (compiled[0] == 0) {
210            Log.e(TAG, "Could not compile shader " + shaderType + ":");
211            Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
212            GLES20.glDeleteShader(shader);
213            // TODO: handle this more gracefully
214            throw new IllegalStateException("Could not compile shader " + shaderType);
215        }
216        return shader;
217    }
218
219    private int createProgram(String vertexSource, String fragmentSource) {
220        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
221        if (vertexShader == 0) {
222            return 0;
223        }
224        int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
225        if (pixelShader == 0) {
226            return 0;
227        }
228
229        int program = GLES20.glCreateProgram();
230        checkGlError("glCreateProgram");
231        if (program == 0) {
232            Log.e(TAG, "Could not create program");
233        }
234        GLES20.glAttachShader(program, vertexShader);
235        checkGlError("glAttachShader");
236        GLES20.glAttachShader(program, pixelShader);
237        checkGlError("glAttachShader");
238        GLES20.glLinkProgram(program);
239        int[] linkStatus = new int[1];
240        GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
241        if (linkStatus[0] != GLES20.GL_TRUE) {
242            Log.e(TAG, "Could not link program: ");
243            Log.e(TAG, GLES20.glGetProgramInfoLog(program));
244            GLES20.glDeleteProgram(program);
245            // TODO: handle this more gracefully
246            throw new IllegalStateException("Could not link program");
247        }
248        return program;
249    }
250
251    private void drawFrame(SurfaceTexture st, int width, int height, int flipType) {
252        checkGlError("onDrawFrame start");
253        st.getTransformMatrix(mSTMatrix);
254
255        Matrix.setIdentityM(mMVPMatrix, /*smOffset*/0);
256
257        // Find intermediate buffer dimensions
258        Size dimens;
259        try {
260            dimens = LegacyCameraDevice.getTextureSize(st);
261        } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
262            // Should never hit this.
263            throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e);
264        }
265        float texWidth = dimens.getWidth();
266        float texHeight = dimens.getHeight();
267
268        if (texWidth <= 0 || texHeight <= 0) {
269            throw new IllegalStateException("Illegal intermediate texture with dimension of 0");
270        }
271
272        // Letterbox or pillar-box output dimensions into intermediate dimensions.
273        RectF intermediate = new RectF(/*left*/0, /*top*/0, /*right*/texWidth, /*bottom*/texHeight);
274        RectF output = new RectF(/*left*/0, /*top*/0, /*right*/width, /*bottom*/height);
275        android.graphics.Matrix boxingXform = new android.graphics.Matrix();
276        boxingXform.setRectToRect(output, intermediate, android.graphics.Matrix.ScaleToFit.CENTER);
277        boxingXform.mapRect(output);
278
279        // Find scaling factor from pillar-boxed/letter-boxed output dimensions to intermediate
280        // buffer dimensions.
281        float scaleX = intermediate.width() / output.width();
282        float scaleY = intermediate.height() / output.height();
283
284        // Intermediate texture is implicitly scaled to 'fill' the output dimensions in clip space
285        // coordinates in the shader.  To avoid stretching, we need to scale the larger dimension
286        // of the intermediate buffer so that the output buffer is actually letter-boxed
287        // or pillar-boxed into the intermediate buffer after clipping.
288        Matrix.scaleM(mMVPMatrix, /*offset*/0, /*x*/scaleX, /*y*/scaleY, /*z*/1);
289
290        if (DEBUG) {
291            Log.d(TAG, "Scaling factors (S_x = " + scaleX + ",S_y = " + scaleY + ") used for " +
292                    width + "x" + height + " surface, intermediate buffer size is " + texWidth +
293                    "x" + texHeight);
294        }
295
296        // Set viewport to be output buffer dimensions
297        GLES20.glViewport(0, 0, width, height);
298
299        if (DEBUG) {
300            GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
301            GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
302        }
303
304        GLES20.glUseProgram(mProgram);
305        checkGlError("glUseProgram");
306
307        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
308        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
309
310        FloatBuffer triangleVertices;
311        switch(flipType) {
312            case FLIP_TYPE_HORIZONTAL:
313                triangleVertices = mHorizontalFlipTriangleVertices;
314                break;
315            case FLIP_TYPE_VERTICAL:
316                triangleVertices = mVerticalFlipTriangleVertices;
317                break;
318            case FLIP_TYPE_BOTH:
319                triangleVertices = mBothFlipTriangleVertices;
320                break;
321            default:
322                triangleVertices = mRegularTriangleVertices;
323                break;
324        }
325
326        triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
327        GLES20.glVertexAttribPointer(maPositionHandle, VERTEX_POS_SIZE, GLES20.GL_FLOAT,
328                /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
329        checkGlError("glVertexAttribPointer maPosition");
330        GLES20.glEnableVertexAttribArray(maPositionHandle);
331        checkGlError("glEnableVertexAttribArray maPositionHandle");
332
333        triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
334        GLES20.glVertexAttribPointer(maTextureHandle, VERTEX_UV_SIZE, GLES20.GL_FLOAT,
335                /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
336        checkGlError("glVertexAttribPointer maTextureHandle");
337        GLES20.glEnableVertexAttribArray(maTextureHandle);
338        checkGlError("glEnableVertexAttribArray maTextureHandle");
339
340        GLES20.glUniformMatrix4fv(muMVPMatrixHandle, /*count*/ 1, /*transpose*/ false, mMVPMatrix,
341                /*offset*/ 0);
342        GLES20.glUniformMatrix4fv(muSTMatrixHandle, /*count*/ 1, /*transpose*/ false, mSTMatrix,
343                /*offset*/ 0);
344
345        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*offset*/ 0, /*count*/ 4);
346        checkGlError("glDrawArrays");
347    }
348
349    /**
350     * Initializes GL state.  Call this after the EGL surface has been created and made current.
351     */
352    private void initializeGLState() {
353        mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
354        if (mProgram == 0) {
355            throw new IllegalStateException("failed creating program");
356        }
357        maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
358        checkGlError("glGetAttribLocation aPosition");
359        if (maPositionHandle == -1) {
360            throw new IllegalStateException("Could not get attrib location for aPosition");
361        }
362        maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
363        checkGlError("glGetAttribLocation aTextureCoord");
364        if (maTextureHandle == -1) {
365            throw new IllegalStateException("Could not get attrib location for aTextureCoord");
366        }
367
368        muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
369        checkGlError("glGetUniformLocation uMVPMatrix");
370        if (muMVPMatrixHandle == -1) {
371            throw new IllegalStateException("Could not get attrib location for uMVPMatrix");
372        }
373
374        muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
375        checkGlError("glGetUniformLocation uSTMatrix");
376        if (muSTMatrixHandle == -1) {
377            throw new IllegalStateException("Could not get attrib location for uSTMatrix");
378        }
379
380        int[] textures = new int[1];
381        GLES20.glGenTextures(/*n*/ 1, textures, /*offset*/ 0);
382
383        mTextureID = textures[0];
384        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
385        checkGlError("glBindTexture mTextureID");
386
387        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
388                GLES20.GL_NEAREST);
389        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
390                GLES20.GL_LINEAR);
391        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
392                GLES20.GL_CLAMP_TO_EDGE);
393        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
394                GLES20.GL_CLAMP_TO_EDGE);
395        checkGlError("glTexParameter");
396    }
397
398    private int getTextureId() {
399        return mTextureID;
400    }
401
402    private void clearState() {
403        mSurfaces.clear();
404        for (EGLSurfaceHolder holder : mConversionSurfaces) {
405            try {
406                LegacyCameraDevice.disconnectSurface(holder.surface);
407            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
408                Log.w(TAG, "Surface abandoned, skipping...", e);
409            }
410        }
411        mConversionSurfaces.clear();
412        mPBufferPixels = null;
413        if (mSurfaceTexture != null) {
414            mSurfaceTexture.release();
415        }
416        mSurfaceTexture = null;
417    }
418
419    private void configureEGLContext() {
420        mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
421        if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
422            throw new IllegalStateException("No EGL14 display");
423        }
424        int[] version = new int[2];
425        if (!EGL14.eglInitialize(mEGLDisplay, version, /*offset*/ 0, version, /*offset*/ 1)) {
426            throw new IllegalStateException("Cannot initialize EGL14");
427        }
428
429        int[] attribList = {
430                EGL14.EGL_RED_SIZE, EGL_COLOR_BITLENGTH,
431                EGL14.EGL_GREEN_SIZE, EGL_COLOR_BITLENGTH,
432                EGL14.EGL_BLUE_SIZE, EGL_COLOR_BITLENGTH,
433                EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
434                EGL_RECORDABLE_ANDROID, 1,
435                EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT | EGL14.EGL_WINDOW_BIT,
436                EGL14.EGL_NONE
437        };
438        EGLConfig[] configs = new EGLConfig[1];
439        int[] numConfigs = new int[1];
440        EGL14.eglChooseConfig(mEGLDisplay, attribList, /*offset*/ 0, configs, /*offset*/ 0,
441                configs.length, numConfigs, /*offset*/ 0);
442        checkEglError("eglCreateContext RGB888+recordable ES2");
443        mConfigs = configs[0];
444        int[] attrib_list = {
445                EGL14.EGL_CONTEXT_CLIENT_VERSION, GLES_VERSION,
446                EGL14.EGL_NONE
447        };
448        mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
449                attrib_list, /*offset*/ 0);
450        checkEglError("eglCreateContext");
451        if(mEGLContext == EGL14.EGL_NO_CONTEXT) {
452            throw new IllegalStateException("No EGLContext could be made");
453        }
454    }
455
456    private void configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces) {
457        if (surfaces == null || surfaces.size() == 0) {
458            throw new IllegalStateException("No Surfaces were provided to draw to");
459        }
460        int[] surfaceAttribs = {
461                EGL14.EGL_NONE
462        };
463        for (EGLSurfaceHolder holder : surfaces) {
464            holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
465                    holder.surface, surfaceAttribs, /*offset*/ 0);
466            checkEglError("eglCreateWindowSurface");
467        }
468    }
469
470    private void configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces) {
471        if (surfaces == null || surfaces.size() == 0) {
472            throw new IllegalStateException("No Surfaces were provided to draw to");
473        }
474
475        int maxLength = 0;
476        for (EGLSurfaceHolder holder : surfaces) {
477            int length = holder.width * holder.height;
478            // Find max surface size, ensure PBuffer can hold this many pixels
479            maxLength = (length > maxLength) ? length : maxLength;
480            int[] surfaceAttribs = {
481                    EGL14.EGL_WIDTH, holder.width,
482                    EGL14.EGL_HEIGHT, holder.height,
483                    EGL14.EGL_NONE
484            };
485            holder.eglSurface =
486                    EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
487            checkEglError("eglCreatePbufferSurface");
488        }
489        mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES)
490                .order(ByteOrder.nativeOrder());
491    }
492
493    private void releaseEGLContext() {
494        if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
495            EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
496                    EGL14.EGL_NO_CONTEXT);
497            dumpGlTiming();
498            if (mSurfaces != null) {
499                for (EGLSurfaceHolder holder : mSurfaces) {
500                    if (holder.eglSurface != null) {
501                        EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
502                    }
503                }
504            }
505            if (mConversionSurfaces != null) {
506                for (EGLSurfaceHolder holder : mConversionSurfaces) {
507                    if (holder.eglSurface != null) {
508                        EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
509                    }
510                }
511            }
512            EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
513            EGL14.eglReleaseThread();
514            EGL14.eglTerminate(mEGLDisplay);
515        }
516
517        mConfigs = null;
518        mEGLDisplay = EGL14.EGL_NO_DISPLAY;
519        mEGLContext = EGL14.EGL_NO_CONTEXT;
520        clearState();
521    }
522
523    private void makeCurrent(EGLSurface surface) {
524        EGL14.eglMakeCurrent(mEGLDisplay, surface, surface, mEGLContext);
525        checkEglError("makeCurrent");
526    }
527
528    private boolean swapBuffers(EGLSurface surface)
529            throws LegacyExceptionUtils.BufferQueueAbandonedException {
530        boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface);
531        int error = EGL14.eglGetError();
532        if (error == EGL14.EGL_BAD_SURFACE) {
533            throw new LegacyExceptionUtils.BufferQueueAbandonedException();
534        } else if (error != EGL14.EGL_SUCCESS) {
535            throw new IllegalStateException("swapBuffers: EGL error: 0x" +
536                    Integer.toHexString(error));
537        }
538        return result;
539    }
540
541    private void checkEglError(String msg) {
542        int error;
543        if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
544            throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error));
545        }
546    }
547
548    private void checkGlError(String msg) {
549        int error;
550        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
551            throw new IllegalStateException(msg + ": GLES20 error: 0x" + Integer.toHexString(error));
552        }
553    }
554
555    /**
556     * Save a measurement dump to disk, in
557     * {@code /sdcard/CameraLegacy/durations_<time>_<width1>x<height1>_...txt}
558     */
559    private void dumpGlTiming() {
560        if (mPerfMeasurer == null) return;
561
562        File legacyStorageDir = new File(Environment.getExternalStorageDirectory(), "CameraLegacy");
563        if (!legacyStorageDir.exists()){
564            if (!legacyStorageDir.mkdirs()){
565                Log.e(TAG, "Failed to create directory for data dump");
566                return;
567            }
568        }
569
570        StringBuilder path = new StringBuilder(legacyStorageDir.getPath());
571        path.append(File.separator);
572        path.append("durations_");
573
574        Time now = new Time();
575        now.setToNow();
576        path.append(now.format2445());
577        path.append("_S");
578        for (EGLSurfaceHolder surface : mSurfaces) {
579            path.append(String.format("_%d_%d", surface.width, surface.height));
580        }
581        path.append("_C");
582        for (EGLSurfaceHolder surface : mConversionSurfaces) {
583            path.append(String.format("_%d_%d", surface.width, surface.height));
584        }
585        path.append(".txt");
586        mPerfMeasurer.dumpPerformanceData(path.toString());
587    }
588
589    private void setupGlTiming() {
590        if (PerfMeasurement.isGlTimingSupported()) {
591            Log.d(TAG, "Enabling GL performance measurement");
592            mPerfMeasurer = new PerfMeasurement();
593        } else {
594            Log.d(TAG, "GL performance measurement not supported on this device");
595            mPerfMeasurer = null;
596        }
597    }
598
599    private void beginGlTiming() {
600        if (mPerfMeasurer == null) return;
601        mPerfMeasurer.startTimer();
602    }
603
604    private void addGlTimestamp(long timestamp) {
605        if (mPerfMeasurer == null) return;
606        mPerfMeasurer.addTimestamp(timestamp);
607    }
608
609    private void endGlTiming() {
610        if (mPerfMeasurer == null) return;
611        mPerfMeasurer.stopTimer();
612    }
613
614    /**
615     * Return the surface texture to draw to - this is the texture use to when producing output
616     * surface buffers.
617     *
618     * @return a {@link SurfaceTexture}.
619     */
620    public SurfaceTexture getSurfaceTexture() {
621        return mSurfaceTexture;
622    }
623
624    /**
625     * Set a collection of output {@link Surface}s that can be drawn to.
626     *
627     * @param surfaces a {@link Collection} of surfaces.
628     */
629    public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) {
630        releaseEGLContext();
631
632        if (surfaces == null || surfaces.size() == 0) {
633            Log.w(TAG, "No output surfaces configured for GL drawing.");
634            return;
635        }
636
637        for (Pair<Surface, Size> p : surfaces) {
638            Surface s = p.first;
639            Size surfaceSize = p.second;
640            // If pixel conversions aren't handled by egl, use a pbuffer
641            try {
642                EGLSurfaceHolder holder = new EGLSurfaceHolder();
643                holder.surface = s;
644                holder.width = surfaceSize.getWidth();
645                holder.height = surfaceSize.getHeight();
646                if (LegacyCameraDevice.needsConversion(s)) {
647                    mConversionSurfaces.add(holder);
648                    // LegacyCameraDevice is the producer of surfaces if it's not handled by EGL,
649                    // so LegacyCameraDevice needs to connect to the surfaces.
650                    LegacyCameraDevice.connectSurface(s);
651                } else {
652                    mSurfaces.add(holder);
653                }
654            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
655                Log.w(TAG, "Surface abandoned, skipping configuration... ", e);
656            }
657        }
658
659        // Set up egl display
660        configureEGLContext();
661
662        // Set up regular egl surfaces if needed
663        if (mSurfaces.size() > 0) {
664            configureEGLOutputSurfaces(mSurfaces);
665        }
666
667        // Set up pbuffer surface if needed
668        if (mConversionSurfaces.size() > 0) {
669            configureEGLPbufferSurfaces(mConversionSurfaces);
670        }
671        makeCurrent((mSurfaces.size() > 0) ? mSurfaces.get(0).eglSurface :
672                mConversionSurfaces.get(0).eglSurface);
673        initializeGLState();
674        mSurfaceTexture = new SurfaceTexture(getTextureId());
675
676        // Set up performance tracking if enabled
677        if (SystemProperties.getBoolean(LEGACY_PERF_PROPERTY, false)) {
678            setupGlTiming();
679        }
680    }
681
682    /**
683     * Draw the current buffer in the {@link SurfaceTexture} returned from
684     * {@link #getSurfaceTexture()} into the set of target {@link Surface}s
685     * in the next request from the given {@link CaptureCollector}, or drop
686     * the frame if none is available.
687     *
688     * <p>
689     * Any {@link Surface}s targeted must be a subset of the {@link Surface}s
690     * set in the last {@link #configureSurfaces(java.util.Collection)} call.
691     * </p>
692     *
693     * @param targetCollector the surfaces to draw to.
694     */
695    public void drawIntoSurfaces(CaptureCollector targetCollector) {
696        if ((mSurfaces == null || mSurfaces.size() == 0)
697                && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) {
698            return;
699        }
700
701        boolean doTiming = targetCollector.hasPendingPreviewCaptures();
702        checkGlError("before updateTexImage");
703
704        if (doTiming) {
705            beginGlTiming();
706        }
707
708        mSurfaceTexture.updateTexImage();
709
710        long timestamp = mSurfaceTexture.getTimestamp();
711
712        Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp);
713
714        // No preview request queued, drop frame.
715        if (captureHolder == null) {
716            if (DEBUG) {
717                Log.d(TAG, "Dropping preview frame.");
718            }
719            if (doTiming) {
720                endGlTiming();
721            }
722            return;
723        }
724
725        RequestHolder request = captureHolder.first;
726
727        Collection<Surface> targetSurfaces = request.getHolderTargets();
728        if (doTiming) {
729            addGlTimestamp(timestamp);
730        }
731
732        List<Long> targetSurfaceIds = new ArrayList();
733        try {
734            targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
735        } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
736            Log.w(TAG, "Surface abandoned, dropping frame. ", e);
737            request.setOutputAbandoned();
738        }
739
740        for (EGLSurfaceHolder holder : mSurfaces) {
741            if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
742                try{
743                    LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
744                            holder.height);
745                    makeCurrent(holder.eglSurface);
746
747                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
748                    drawFrame(mSurfaceTexture, holder.width, holder.height,
749                            (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
750                                    FLIP_TYPE_HORIZONTAL : FLIP_TYPE_NONE);
751                    swapBuffers(holder.eglSurface);
752                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
753                    Log.w(TAG, "Surface abandoned, dropping frame. ", e);
754                    request.setOutputAbandoned();
755                }
756            }
757        }
758        for (EGLSurfaceHolder holder : mConversionSurfaces) {
759            if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
760                makeCurrent(holder.eglSurface);
761                // glReadPixels reads from the bottom of the buffer, so add an extra vertical flip
762                drawFrame(mSurfaceTexture, holder.width, holder.height,
763                        (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
764                                FLIP_TYPE_BOTH : FLIP_TYPE_VERTICAL);
765                mPBufferPixels.clear();
766                GLES20.glReadPixels(/*x*/ 0, /*y*/ 0, holder.width, holder.height,
767                        GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPBufferPixels);
768                checkGlError("glReadPixels");
769
770                try {
771                    int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
772                    LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
773                            holder.height);
774                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
775                    LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
776                            holder.width, holder.height, format);
777                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
778                    Log.w(TAG, "Surface abandoned, dropping frame. ", e);
779                    request.setOutputAbandoned();
780                }
781            }
782        }
783        targetCollector.previewProduced();
784
785        if (doTiming) {
786            endGlTiming();
787        }
788    }
789
790    /**
791     * Clean up the current GL context.
792     */
793    public void cleanupEGLContext() {
794        releaseEGLContext();
795    }
796
797    /**
798     * Drop all current GL operations on the floor.
799     */
800    public void flush() {
801        // TODO: implement flush
802        Log.e(TAG, "Flush not yet implemented.");
803    }
804}
805