SurfaceTexture.java revision c5f94d8a4779050125145396ca83fbc862c7ed6b
16714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis/* 26714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Copyright (C) 2010 The Android Open Source Project 36714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 46714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Licensed under the Apache License, Version 2.0 (the "License"); 56714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * you may not use this file except in compliance with the License. 66714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * You may obtain a copy of the License at 76714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 86714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * http://www.apache.org/licenses/LICENSE-2.0 96714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 106714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Unless required by applicable law or agreed to in writing, software 116714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * distributed under the License is distributed on an "AS IS" BASIS, 126714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * See the License for the specific language governing permissions and 146714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * limitations under the License. 156714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 166714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 176714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennispackage android.graphics; 186714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 19376590d668e22a918439877b55faf075427b13f3Jamie Gennisimport java.lang.ref.WeakReference; 20376590d668e22a918439877b55faf075427b13f3Jamie Gennisimport android.os.Handler; 21376590d668e22a918439877b55faf075427b13f3Jamie Gennisimport android.os.Looper; 22376590d668e22a918439877b55faf075427b13f3Jamie Gennisimport android.os.Message; 23376590d668e22a918439877b55faf075427b13f3Jamie Gennis 246714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis/** 256714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Captures frames from an image stream as an OpenGL ES texture. 266714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 27cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten * <p>The image stream may come from either camera preview or video decode. A SurfaceTexture 28cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten * may be used in place of a SurfaceHolder when specifying the output destination of a 29cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten * {@link android.hardware.Camera} or {@link android.media.MediaPlayer} 3037cec0fc50760fe89614863eded14011f9412534Jamie Gennis * object. Doing so will cause all the frames from the image stream to be sent to the 316714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * SurfaceTexture object rather than to the device's display. When {@link #updateTexImage} is 32cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten * called, the contents of the texture object specified when the SurfaceTexture was created are 336714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * updated to contain the most recent image from the image stream. This may cause some frames of 346714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * the stream to be skipped. 356714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 365f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * <p>When sampling from the texture one should first transform the texture coordinates using the 375f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * matrix queried via {@link #getTransformMatrix}. The transform matrix may change each time {@link 385f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * #updateTexImage} is called, so it should be re-queried each time the texture image is updated. 395f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s, 405f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in 415f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * the streamed texture. This transform compensates for any properties of the image stream source 425f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * that cause it to appear different from a traditional OpenGL ES texture. For example, sampling 435f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * from the bottom left corner of the image can be accomplished by transforming the column vector 445f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can 455f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * be done by transforming (1, 1, 0, 1). 465f8b6653e0b5b99097db7d8f41d5251f7b398704Jamie Gennis * 476714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the 48858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt"> 49858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * GL_OES_EGL_image_external</a> OpenGL ES extension. This limits how the texture may be used. 50858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than 51858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture 52858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * must declare its use of this extension using, for example, an "#extension 53858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using 54858873271250e80a704c29c71ff0578a87bd9d31Jamie Gennis * the samplerExternalOES GLSL sampler type. 5537cec0fc50760fe89614863eded14011f9412534Jamie Gennis * 5637cec0fc50760fe89614863eded14011f9412534Jamie Gennis * <p>SurfaceTexture objects may be created on any thread. {@link #updateTexImage} may only be 5737cec0fc50760fe89614863eded14011f9412534Jamie Gennis * called on the thread with the OpenGL ES context that contains the texture object. The 5837cec0fc50760fe89614863eded14011f9412534Jamie Gennis * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link 5937cec0fc50760fe89614863eded14011f9412534Jamie Gennis * #updateTexImage} should not be called directly from the callback. 606714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 616714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennispublic class SurfaceTexture { 626714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 63376590d668e22a918439877b55faf075427b13f3Jamie Gennis private EventHandler mEventHandler; 64376590d668e22a918439877b55faf075427b13f3Jamie Gennis private OnFrameAvailableListener mOnFrameAvailableListener; 65376590d668e22a918439877b55faf075427b13f3Jamie Gennis 666714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis @SuppressWarnings("unused") 676714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis private int mSurfaceTexture; 686714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 696714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /** 706714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Callback interface for being notified that a new stream frame is available. 716714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 726714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public interface OnFrameAvailableListener { 736714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis void onFrameAvailable(SurfaceTexture surfaceTexture); 746714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 756714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 766714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /** 776714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Exception thrown when a surface couldn't be created or resized 786714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 796714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public static class OutOfResourcesException extends Exception { 806714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public OutOfResourcesException() { 816714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 826714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public OutOfResourcesException(String name) { 836714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis super(name); 846714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 856714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 866714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 876714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /** 886714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 896714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * 906714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 916714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 926714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public SurfaceTexture(int texName) { 93376590d668e22a918439877b55faf075427b13f3Jamie Gennis Looper looper; 94376590d668e22a918439877b55faf075427b13f3Jamie Gennis if ((looper = Looper.myLooper()) != null) { 95376590d668e22a918439877b55faf075427b13f3Jamie Gennis mEventHandler = new EventHandler(looper); 96376590d668e22a918439877b55faf075427b13f3Jamie Gennis } else if ((looper = Looper.getMainLooper()) != null) { 97376590d668e22a918439877b55faf075427b13f3Jamie Gennis mEventHandler = new EventHandler(looper); 98376590d668e22a918439877b55faf075427b13f3Jamie Gennis } else { 99376590d668e22a918439877b55faf075427b13f3Jamie Gennis mEventHandler = null; 100376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 101376590d668e22a918439877b55faf075427b13f3Jamie Gennis nativeInit(texName, new WeakReference<SurfaceTexture>(this)); 1026714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 1036714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 1046714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /** 1056714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Register a callback to be invoked when a new image frame becomes available to the 1066714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * SurfaceTexture. Note that this callback may be called on an arbitrary thread, so it is not 1076714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the 1086714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * thread invoking the callback. 1096714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 1106714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis public void setOnFrameAvailableListener(OnFrameAvailableListener l) { 111376590d668e22a918439877b55faf075427b13f3Jamie Gennis mOnFrameAvailableListener = l; 1126714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis } 1136714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 1146714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /** 1156714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * Update the texture image to the most recent frame from the image stream. This may only be 1166714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * called while the OpenGL ES context that owns the texture is bound to the thread. It will 1176714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target. 1186714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 119376590d668e22a918439877b55faf075427b13f3Jamie Gennis public void updateTexImage() { 120376590d668e22a918439877b55faf075427b13f3Jamie Gennis nativeUpdateTexImage(); 121376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 122b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis 123b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis /** 124b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by 125b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * the most recent call to updateTexImage. 126b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * 127b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s 128b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample 129b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * that location from the texture. Sampling the texture outside of the range of this transform 130b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * is undefined. 131b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * 132b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via 133b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * the glLoadMatrixf or glUniformMatrix4fv functions. 134b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * 135b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * @param mtx the array into which the 4x4 matrix will be stored. The array must have exactly 136b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis * 16 elements. 137b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis */ 138b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis public void getTransformMatrix(float[] mtx) { 139cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten // Note we intentionally don't check mtx for null, so this will result in a 140cc562a3576a6a8096626387472e05e8bee03352aGlenn Kasten // NullPointerException. But it's safe because it happens before the call to native. 141b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis if (mtx.length != 16) { 142b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis throw new IllegalArgumentException(); 143b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis } 144376590d668e22a918439877b55faf075427b13f3Jamie Gennis nativeGetTransformMatrix(mtx); 145b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis } 146b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis 147c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala /** 148c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * Retrieve the timestamp associated with the texture image set by the most recent call to 149c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * updateTexImage. 150c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * 151c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * This timestamp is in nanoseconds, and is guaranteed to be monotonically increasing. The 152c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * specific meaning and zero point of the timestamp depends on the source providing images to 153c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * the SurfaceTexture. Unless otherwise specified by the image source, timestamps cannot 154c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * generally be compared across SurfaceTexture instances, or across multiple program 155c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * invocations. It is mostly useful for determining time offsets between subsequent frames. 156c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala * @hide 157c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala */ 158c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala public long getTimestamp() { 159c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala return nativeGetTimestamp(); 160c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala } 161c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala 162376590d668e22a918439877b55faf075427b13f3Jamie Gennis protected void finalize() throws Throwable { 163376590d668e22a918439877b55faf075427b13f3Jamie Gennis try { 164376590d668e22a918439877b55faf075427b13f3Jamie Gennis nativeFinalize(); 165376590d668e22a918439877b55faf075427b13f3Jamie Gennis } finally { 166376590d668e22a918439877b55faf075427b13f3Jamie Gennis super.finalize(); 167376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 168376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 169376590d668e22a918439877b55faf075427b13f3Jamie Gennis 170376590d668e22a918439877b55faf075427b13f3Jamie Gennis private class EventHandler extends Handler { 171376590d668e22a918439877b55faf075427b13f3Jamie Gennis public EventHandler(Looper looper) { 172376590d668e22a918439877b55faf075427b13f3Jamie Gennis super(looper); 173376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 174376590d668e22a918439877b55faf075427b13f3Jamie Gennis 175376590d668e22a918439877b55faf075427b13f3Jamie Gennis @Override 176376590d668e22a918439877b55faf075427b13f3Jamie Gennis public void handleMessage(Message msg) { 177376590d668e22a918439877b55faf075427b13f3Jamie Gennis if (mOnFrameAvailableListener != null) { 178376590d668e22a918439877b55faf075427b13f3Jamie Gennis mOnFrameAvailableListener.onFrameAvailable(SurfaceTexture.this); 179376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 180376590d668e22a918439877b55faf075427b13f3Jamie Gennis return; 181376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 182376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 183376590d668e22a918439877b55faf075427b13f3Jamie Gennis 184376590d668e22a918439877b55faf075427b13f3Jamie Gennis private static void postEventFromNative(Object selfRef) { 185376590d668e22a918439877b55faf075427b13f3Jamie Gennis WeakReference weakSelf = (WeakReference)selfRef; 186376590d668e22a918439877b55faf075427b13f3Jamie Gennis SurfaceTexture st = (SurfaceTexture)weakSelf.get(); 187376590d668e22a918439877b55faf075427b13f3Jamie Gennis if (st == null) { 188376590d668e22a918439877b55faf075427b13f3Jamie Gennis return; 189376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 190376590d668e22a918439877b55faf075427b13f3Jamie Gennis 191376590d668e22a918439877b55faf075427b13f3Jamie Gennis if (st.mEventHandler != null) { 192376590d668e22a918439877b55faf075427b13f3Jamie Gennis Message m = st.mEventHandler.obtainMessage(); 193376590d668e22a918439877b55faf075427b13f3Jamie Gennis st.mEventHandler.sendMessage(m); 194376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 195376590d668e22a918439877b55faf075427b13f3Jamie Gennis } 196b0ba48c95ea8768a051100c5adb4c906caa1e080Jamie Gennis 197376590d668e22a918439877b55faf075427b13f3Jamie Gennis private native void nativeInit(int texName, Object weakSelf); 198376590d668e22a918439877b55faf075427b13f3Jamie Gennis private native void nativeFinalize(); 199376590d668e22a918439877b55faf075427b13f3Jamie Gennis private native void nativeGetTransformMatrix(float[] mtx); 200c5f94d8a4779050125145396ca83fbc862c7ed6bEino-Ville Talvala private native long nativeGetTimestamp(); 201376590d668e22a918439877b55faf075427b13f3Jamie Gennis private native void nativeUpdateTexImage(); 2026714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis 2036714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis /* 2046714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * We use a class initializer to allow the native code to cache some 2056714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis * field offsets. 2066714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis */ 2076714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis private static native void nativeClassInit(); 2086714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis static { nativeClassInit(); } 2096714efc5e0c52953b65e774de0003e22377e7d39Jamie Gennis} 210