SurfaceTexture.java revision ba873d21718b0c0bbeabbb3b4796e54073739700
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.graphics; 18 19import java.lang.ref.WeakReference; 20 21import android.annotation.Nullable; 22import android.os.Handler; 23import android.os.Looper; 24import android.os.Message; 25import android.view.Surface; 26 27/** 28 * Captures frames from an image stream as an OpenGL ES texture. 29 * 30 * <p>The image stream may come from either camera preview or video decode. A SurfaceTexture 31 * may be used in place of a SurfaceHolder when specifying the output destination of a 32 * {@link android.hardware.Camera} or {@link android.media.MediaPlayer} 33 * object. Doing so will cause all the frames from the image stream to be sent to the 34 * SurfaceTexture object rather than to the device's display. When {@link #updateTexImage} is 35 * called, the contents of the texture object specified when the SurfaceTexture was created are 36 * updated to contain the most recent image from the image stream. This may cause some frames of 37 * the stream to be skipped. 38 * 39 * <p>When sampling from the texture one should first transform the texture coordinates using the 40 * matrix queried via {@link #getTransformMatrix(float[])}. The transform matrix may change each 41 * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image 42 * is updated. 43 * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s, 44 * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in 45 * the streamed texture. This transform compensates for any properties of the image stream source 46 * that cause it to appear different from a traditional OpenGL ES texture. For example, sampling 47 * from the bottom left corner of the image can be accomplished by transforming the column vector 48 * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can 49 * be done by transforming (1, 1, 0, 1). 50 * 51 * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the 52 * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt"> 53 * GL_OES_EGL_image_external</a> OpenGL ES extension. This limits how the texture may be used. 54 * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than 55 * the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture 56 * must declare its use of this extension using, for example, an "#extension 57 * GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using 58 * the samplerExternalOES GLSL sampler type. 59 * 60 * <p>SurfaceTexture objects may be created on any thread. {@link #updateTexImage} may only be 61 * called on the thread with the OpenGL ES context that contains the texture object. The 62 * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link 63 * #updateTexImage} should not be called directly from the callback. 64 */ 65public class SurfaceTexture { 66 private final Looper mCreatorLooper; 67 private Handler mOnFrameAvailableHandler; 68 69 /** 70 * These fields are used by native code, do not access or modify. 71 */ 72 private long mSurfaceTexture; 73 private long mProducer; 74 private long mFrameAvailableListener; 75 76 /** 77 * Callback interface for being notified that a new stream frame is available. 78 */ 79 public interface OnFrameAvailableListener { 80 void onFrameAvailable(SurfaceTexture surfaceTexture); 81 } 82 83 /** 84 * Exception thrown when a SurfaceTexture couldn't be created or resized. 85 * 86 * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException} 87 * is used instead. 88 */ 89 @SuppressWarnings("serial") 90 @Deprecated 91 public static class OutOfResourcesException extends Exception { 92 public OutOfResourcesException() { 93 } 94 public OutOfResourcesException(String name) { 95 super(name); 96 } 97 } 98 99 /** 100 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 101 * 102 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 103 * 104 * @throws Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 105 */ 106 public SurfaceTexture(int texName) { 107 this(texName, false); 108 } 109 110 /** 111 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 112 * 113 * In single buffered mode the application is responsible for serializing access to the image 114 * content buffer. Each time the image content is to be updated, the 115 * {@link #releaseTexImage()} method must be called before the image content producer takes 116 * ownership of the buffer. For example, when producing image content with the NDK 117 * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()} 118 * must be called before each ANativeWindow_lock, or that call will fail. When producing 119 * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first 120 * OpenGL ES function call each frame. 121 * 122 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 123 * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode. 124 * 125 * @throws Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 126 */ 127 public SurfaceTexture(int texName, boolean singleBufferMode) { 128 mCreatorLooper = Looper.myLooper(); 129 nativeInit(texName, singleBufferMode, new WeakReference<SurfaceTexture>(this)); 130 } 131 132 /** 133 * Register a callback to be invoked when a new image frame becomes available to the 134 * SurfaceTexture. 135 * <p> 136 * The callback may be called on an arbitrary thread, so it is not 137 * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the 138 * thread invoking the callback. 139 * </p> 140 * 141 * @param listener The listener to use, or null to remove the listener. 142 */ 143 public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) { 144 setOnFrameAvailableListener(listener, null); 145 } 146 147 /** 148 * Register a callback to be invoked when a new image frame becomes available to the 149 * SurfaceTexture. 150 * <p> 151 * If a handler is specified, the callback will be invoked on that handler's thread. 152 * If no handler is specified, then the callback may be called on an arbitrary thread, 153 * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES 154 * context to the thread invoking the callback. 155 * </p> 156 * 157 * @param listener The listener to use, or null to remove the listener. 158 * @param handler The handler on which the listener should be invoked, or null 159 * to use an arbitrary thread. 160 */ 161 public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener, 162 @Nullable Handler handler) { 163 if (listener != null) { 164 // Although we claim the thread is arbitrary, earlier implementation would 165 // prefer to send the callback on the creating looper or the main looper 166 // so we preserve this behavior here. 167 Looper looper = handler != null ? handler.getLooper() : 168 mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper(); 169 mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) { 170 @Override 171 public void handleMessage(Message msg) { 172 listener.onFrameAvailable(SurfaceTexture.this); 173 } 174 }; 175 } else { 176 mOnFrameAvailableHandler = null; 177 } 178 } 179 180 /** 181 * Set the default size of the image buffers. The image producer may override the buffer size, 182 * in which case the producer-set buffer size will be used, not the default size set by this 183 * method. Both video and camera based image producers do override the size. This method may 184 * be used to set the image size when producing images with {@link android.graphics.Canvas} (via 185 * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface). 186 * 187 * The new default buffer size will take effect the next time the image producer requests a 188 * buffer to fill. For {@link android.graphics.Canvas} this will be the next time {@link 189 * android.view.Surface#lockCanvas} is called. For OpenGL ES, the EGLSurface should be 190 * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated 191 * (via eglCreateWindowSurface) to ensure that the new default size has taken effect. 192 * 193 * The width and height parameters must be no greater than the minimum of 194 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see 195 * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}). 196 * An error due to invalid dimensions might not be reported until 197 * updateTexImage() is called. 198 */ 199 public void setDefaultBufferSize(int width, int height) { 200 nativeSetDefaultBufferSize(width, height); 201 } 202 203 /** 204 * Update the texture image to the most recent frame from the image stream. This may only be 205 * called while the OpenGL ES context that owns the texture is current on the calling thread. 206 * It will implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target. 207 */ 208 public void updateTexImage() { 209 nativeUpdateTexImage(); 210 } 211 212 /** 213 * Releases the the texture content. This is needed in single buffered mode to allow the image 214 * content producer to take ownership of the image buffer. 215 * For more information see {@link #SurfaceTexture(int, boolean)}. 216 */ 217 public void releaseTexImage() { 218 nativeReleaseTexImage(); 219 } 220 221 /** 222 * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object. 223 * This call must be made with the OpenGL ES context current on the calling thread. The OpenGL 224 * ES texture object will be deleted as a result of this call. After calling this method all 225 * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until 226 * a successful call to {@link #attachToGLContext} is made. 227 * 228 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 229 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 230 * context at a time. 231 */ 232 public void detachFromGLContext() { 233 int err = nativeDetachFromGLContext(); 234 if (err != 0) { 235 throw new RuntimeException("Error during detachFromGLContext (see logcat for details)"); 236 } 237 } 238 239 /** 240 * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread. A 241 * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame 242 * that was current at the time of the last call to {@link #detachFromGLContext}. This new 243 * texture is bound to the GL_TEXTURE_EXTERNAL_OES texture target. 244 * 245 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 246 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 247 * context at a time. 248 * 249 * @param texName The name of the OpenGL ES texture that will be created. This texture name 250 * must be unusued in the OpenGL ES context that is current on the calling thread. 251 */ 252 public void attachToGLContext(int texName) { 253 int err = nativeAttachToGLContext(texName); 254 if (err != 0) { 255 throw new RuntimeException("Error during attachToGLContext (see logcat for details)"); 256 } 257 } 258 259 /** 260 * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by 261 * the most recent call to updateTexImage. 262 * 263 * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s 264 * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample 265 * that location from the texture. Sampling the texture outside of the range of this transform 266 * is undefined. 267 * 268 * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via 269 * the glLoadMatrixf or glUniformMatrix4fv functions. 270 * 271 * @param mtx the array into which the 4x4 matrix will be stored. The array must have exactly 272 * 16 elements. 273 */ 274 public void getTransformMatrix(float[] mtx) { 275 // Note we intentionally don't check mtx for null, so this will result in a 276 // NullPointerException. But it's safe because it happens before the call to native. 277 if (mtx.length != 16) { 278 throw new IllegalArgumentException(); 279 } 280 nativeGetTransformMatrix(mtx); 281 } 282 283 /** 284 * Retrieve the timestamp associated with the texture image set by the most recent call to 285 * updateTexImage. 286 * 287 * This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp 288 * should be unaffected by time-of-day adjustments, and for a camera should be strictly 289 * monotonic but for a MediaPlayer may be reset when the position is set. The 290 * specific meaning and zero point of the timestamp depends on the source providing images to 291 * the SurfaceTexture. Unless otherwise specified by the image source, timestamps cannot 292 * generally be compared across SurfaceTexture instances, or across multiple program 293 * invocations. It is mostly useful for determining time offsets between subsequent frames. 294 */ 295 296 public long getTimestamp() { 297 return nativeGetTimestamp(); 298 } 299 300 /** 301 * release() frees all the buffers and puts the SurfaceTexture into the 302 * 'abandoned' state. Once put in this state the SurfaceTexture can never 303 * leave it. When in the 'abandoned' state, all methods of the 304 * IGraphicBufferProducer interface will fail with the NO_INIT error. 305 * 306 * Note that while calling this method causes all the buffers to be freed 307 * from the perspective of the the SurfaceTexture, if there are additional 308 * references on the buffers (e.g. if a buffer is referenced by a client or 309 * by OpenGL ES as a texture) then those buffer will remain allocated. 310 * 311 * Always call this method when you are done with SurfaceTexture. Failing 312 * to do so may delay resource deallocation for a significant amount of 313 * time. 314 */ 315 public void release() { 316 nativeRelease(); 317 } 318 319 @Override 320 protected void finalize() throws Throwable { 321 try { 322 nativeFinalize(); 323 } finally { 324 super.finalize(); 325 } 326 } 327 328 /** 329 * This method is invoked from native code only. 330 */ 331 @SuppressWarnings({"UnusedDeclaration"}) 332 private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) { 333 SurfaceTexture st = weakSelf.get(); 334 if (st != null) { 335 Handler handler = st.mOnFrameAvailableHandler; 336 if (handler != null) { 337 handler.sendEmptyMessage(0); 338 } 339 } 340 } 341 342 private native void nativeInit(int texName, boolean singleBufferMode, 343 WeakReference<SurfaceTexture> weakSelf) 344 throws Surface.OutOfResourcesException; 345 private native void nativeFinalize(); 346 private native void nativeGetTransformMatrix(float[] mtx); 347 private native long nativeGetTimestamp(); 348 private native void nativeSetDefaultBufferSize(int width, int height); 349 private native void nativeUpdateTexImage(); 350 private native void nativeReleaseTexImage(); 351 private native int nativeDetachFromGLContext(); 352 private native int nativeAttachToGLContext(int texName); 353 private native int nativeGetQueuedCount(); 354 private native void nativeRelease(); 355 356 /* 357 * We use a class initializer to allow the native code to cache some 358 * field offsets. 359 */ 360 private static native void nativeClassInit(); 361 static { nativeClassInit(); } 362} 363