SurfaceTexture.java revision 2b4bfa5efec7df408b4db127961cfc9aca9e57cf
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.graphics; 18 19import java.lang.ref.WeakReference; 20 21import android.os.Handler; 22import android.os.Looper; 23import android.os.Message; 24 25/** 26 * Captures frames from an image stream as an OpenGL ES texture. 27 * 28 * <p>The image stream may come from either camera preview or video decode. A SurfaceTexture 29 * may be used in place of a SurfaceHolder when specifying the output destination of a 30 * {@link android.hardware.Camera} or {@link android.media.MediaPlayer} 31 * object. Doing so will cause all the frames from the image stream to be sent to the 32 * SurfaceTexture object rather than to the device's display. When {@link #updateTexImage} is 33 * called, the contents of the texture object specified when the SurfaceTexture was created are 34 * updated to contain the most recent image from the image stream. This may cause some frames of 35 * the stream to be skipped. 36 * 37 * <p>When sampling from the texture one should first transform the texture coordinates using the 38 * matrix queried via {@link #getTransformMatrix(float[])}. The transform matrix may change each 39 * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image 40 * is updated. 41 * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s, 42 * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in 43 * the streamed texture. This transform compensates for any properties of the image stream source 44 * that cause it to appear different from a traditional OpenGL ES texture. For example, sampling 45 * from the bottom left corner of the image can be accomplished by transforming the column vector 46 * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can 47 * be done by transforming (1, 1, 0, 1). 48 * 49 * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the 50 * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt"> 51 * GL_OES_EGL_image_external</a> OpenGL ES extension. This limits how the texture may be used. 52 * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than 53 * the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture 54 * must declare its use of this extension using, for example, an "#extension 55 * GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using 56 * the samplerExternalOES GLSL sampler type. 57 * 58 * <p>SurfaceTexture objects may be created on any thread. {@link #updateTexImage} may only be 59 * called on the thread with the OpenGL ES context that contains the texture object. The 60 * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link 61 * #updateTexImage} should not be called directly from the callback. 62 */ 63public class SurfaceTexture { 64 65 private EventHandler mEventHandler; 66 private OnFrameAvailableListener mOnFrameAvailableListener; 67 68 /** 69 * This field is used by native code, do not access or modify. 70 */ 71 private int mSurfaceTexture; 72 73 /** 74 * Callback interface for being notified that a new stream frame is available. 75 */ 76 public interface OnFrameAvailableListener { 77 void onFrameAvailable(SurfaceTexture surfaceTexture); 78 } 79 80 /** 81 * Exception thrown when a surface couldn't be created or resized 82 */ 83 public static class OutOfResourcesException extends Exception { 84 public OutOfResourcesException() { 85 } 86 public OutOfResourcesException(String name) { 87 super(name); 88 } 89 } 90 91 /** 92 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 93 * 94 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 95 */ 96 public SurfaceTexture(int texName) { 97 this(texName, false); 98 } 99 100 /** 101 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 102 * 103 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 104 * @param allowSynchronousMode whether the SurfaceTexture can run in the synchronous mode. 105 * When the image stream comes from OpenGL, SurfaceTexture may run in the synchronous 106 * mode where the producer side may be blocked to avoid skipping frames. To avoid the 107 * thread block, set allowSynchronousMode to false. 108 * 109 * @hide 110 */ 111 public SurfaceTexture(int texName, boolean allowSynchronousMode) { 112 Looper looper; 113 if ((looper = Looper.myLooper()) != null) { 114 mEventHandler = new EventHandler(looper); 115 } else if ((looper = Looper.getMainLooper()) != null) { 116 mEventHandler = new EventHandler(looper); 117 } else { 118 mEventHandler = null; 119 } 120 nativeInit(texName, new WeakReference<SurfaceTexture>(this), allowSynchronousMode); 121 } 122 123 /** 124 * Register a callback to be invoked when a new image frame becomes available to the 125 * SurfaceTexture. Note that this callback may be called on an arbitrary thread, so it is not 126 * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the 127 * thread invoking the callback. 128 */ 129 public void setOnFrameAvailableListener(OnFrameAvailableListener l) { 130 mOnFrameAvailableListener = l; 131 } 132 133 /** 134 * Set the default size of the image buffers. The image producer may override the buffer size, 135 * in which case the producer-set buffer size will be used, not the default size set by this 136 * method. Both video and camera based image producers do override the size. This method may 137 * be used to set the image size when producing images with {@link android.graphics.Canvas} (via 138 * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface). 139 * 140 * The new default buffer size will take effect the next time the image producer requests a 141 * buffer to fill. For {@link android.graphics.Canvas} this will be the next time {@link 142 * android.view.Surface#lockCanvas} is called. For OpenGL ES, the EGLSurface should be 143 * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated 144 * (via eglCreateWindowSurface) to ensure that the new default size has taken effect. 145 * 146 * The width and height parameters must be no greater than the minimum of 147 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see 148 * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}). 149 * An error due to invalid dimensions might not be reported until 150 * updateTexImage() is called. 151 */ 152 public void setDefaultBufferSize(int width, int height) { 153 nativeSetDefaultBufferSize(width, height); 154 } 155 156 /** 157 * Update the texture image to the most recent frame from the image stream. This may only be 158 * called while the OpenGL ES context that owns the texture is current on the calling thread. 159 * It will implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target. 160 */ 161 public void updateTexImage() { 162 nativeUpdateTexImage(); 163 } 164 165 /** 166 * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object. 167 * This call must be made with the OpenGL ES context current on the calling thread. The OpenGL 168 * ES texture object will be deleted as a result of this call. After calling this method all 169 * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until 170 * a successful call to {@link #attachToGLContext} is made. 171 * 172 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 173 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 174 * context at a time. 175 * 176 * @hide 177 */ 178 public void detachFromGLContext() { 179 int err = nativeDetachFromGLContext(); 180 if (err != 0) { 181 throw new RuntimeException("Error during detachFromGLContext (see logcat for details)"); 182 } 183 } 184 185 /** 186 * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread. A 187 * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame 188 * that was current at the time of the last call to {@link #detachFromGLContext}. This new 189 * texture is bound to the GL_TEXTURE_EXTERNAL_OES texture target. 190 * 191 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 192 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 193 * context at a time. 194 * 195 * @param texName The name of the OpenGL ES texture that will be created. This texture name 196 * must be unusued in the OpenGL ES context that is current on the calling thread. 197 * 198 * @hide 199 */ 200 public void attachToGLContext(int texName) { 201 int err = nativeAttachToGLContext(texName); 202 if (err != 0) { 203 throw new RuntimeException("Error during detachFromGLContext (see logcat for details)"); 204 } 205 } 206 207 /** 208 * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by 209 * the most recent call to updateTexImage. 210 * 211 * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s 212 * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample 213 * that location from the texture. Sampling the texture outside of the range of this transform 214 * is undefined. 215 * 216 * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via 217 * the glLoadMatrixf or glUniformMatrix4fv functions. 218 * 219 * @param mtx the array into which the 4x4 matrix will be stored. The array must have exactly 220 * 16 elements. 221 */ 222 public void getTransformMatrix(float[] mtx) { 223 // Note we intentionally don't check mtx for null, so this will result in a 224 // NullPointerException. But it's safe because it happens before the call to native. 225 if (mtx.length != 16) { 226 throw new IllegalArgumentException(); 227 } 228 nativeGetTransformMatrix(mtx); 229 } 230 231 /** 232 * Retrieve the timestamp associated with the texture image set by the most recent call to 233 * updateTexImage. 234 * 235 * This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp 236 * should be unaffected by time-of-day adjustments, and for a camera should be strictly 237 * monotonic but for a MediaPlayer may be reset when the position is set. The 238 * specific meaning and zero point of the timestamp depends on the source providing images to 239 * the SurfaceTexture. Unless otherwise specified by the image source, timestamps cannot 240 * generally be compared across SurfaceTexture instances, or across multiple program 241 * invocations. It is mostly useful for determining time offsets between subsequent frames. 242 */ 243 244 public long getTimestamp() { 245 return nativeGetTimestamp(); 246 } 247 248 /** 249 * release() frees all the buffers and puts the SurfaceTexture into the 250 * 'abandoned' state. Once put in this state the SurfaceTexture can never 251 * leave it. When in the 'abandoned' state, all methods of the 252 * ISurfaceTexture interface will fail with the NO_INIT error. 253 * 254 * Note that while calling this method causes all the buffers to be freed 255 * from the perspective of the the SurfaceTexture, if there are additional 256 * references on the buffers (e.g. if a buffer is referenced by a client or 257 * by OpenGL ES as a texture) then those buffer will remain allocated. 258 * 259 * Always call this method when you are done with SurfaceTexture. Failing 260 * to do so may delay resource deallocation for a significant amount of 261 * time. 262 */ 263 public void release() { 264 nativeRelease(); 265 } 266 267 protected void finalize() throws Throwable { 268 try { 269 nativeFinalize(); 270 } finally { 271 super.finalize(); 272 } 273 } 274 275 private class EventHandler extends Handler { 276 public EventHandler(Looper looper) { 277 super(looper); 278 } 279 280 @Override 281 public void handleMessage(Message msg) { 282 if (mOnFrameAvailableListener != null) { 283 mOnFrameAvailableListener.onFrameAvailable(SurfaceTexture.this); 284 } 285 } 286 } 287 288 /** 289 * This method is invoked from native code only. 290 */ 291 @SuppressWarnings({"UnusedDeclaration"}) 292 private static void postEventFromNative(Object selfRef) { 293 WeakReference weakSelf = (WeakReference)selfRef; 294 SurfaceTexture st = (SurfaceTexture)weakSelf.get(); 295 if (st == null) { 296 return; 297 } 298 299 if (st.mEventHandler != null) { 300 Message m = st.mEventHandler.obtainMessage(); 301 st.mEventHandler.sendMessage(m); 302 } 303 } 304 305 private native void nativeInit(int texName, Object weakSelf, boolean allowSynchronousMode); 306 private native void nativeFinalize(); 307 private native void nativeGetTransformMatrix(float[] mtx); 308 private native long nativeGetTimestamp(); 309 private native void nativeSetDefaultBufferSize(int width, int height); 310 private native void nativeUpdateTexImage(); 311 private native int nativeDetachFromGLContext(); 312 private native int nativeAttachToGLContext(int texName); 313 private native int nativeGetQueuedCount(); 314 private native void nativeRelease(); 315 316 /* 317 * We use a class initializer to allow the native code to cache some 318 * field offsets. 319 */ 320 private static native void nativeClassInit(); 321 static { nativeClassInit(); } 322} 323