1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.graphics;
18 
19 import android.annotation.Nullable;
20 import android.compat.annotation.UnsupportedAppUsage;
21 import android.os.Handler;
22 import android.os.Looper;
23 import android.os.Message;
24 import android.view.Surface;
25 
26 import java.lang.ref.WeakReference;
27 
28 /**
29  * Captures frames from an image stream as an OpenGL ES texture.
30  *
31  * <p>The image stream may come from either camera preview or video decode. A
32  * {@link android.view.Surface} created from a SurfaceTexture can be used as an output
33  * destination for the {@link android.hardware.camera2}, {@link android.media.MediaCodec},
34  * {@link android.media.MediaPlayer}, and {@link android.renderscript.Allocation} APIs.
35  * When {@link #updateTexImage} is called, the contents of the texture object specified
36  * when the SurfaceTexture was created are updated to contain the most recent image from the image
37  * stream.  This may cause some frames of the stream to be skipped.
38  *
39  * <p>A SurfaceTexture may also be used in place of a SurfaceHolder when specifying the output
40  * destination of the older {@link android.hardware.Camera} API. Doing so will cause all the
41  * frames from the image stream to be sent to the SurfaceTexture object rather than to the device's
42  * display.
43  *
44  * <p>When sampling from the texture one should first transform the texture coordinates using the
45  * matrix queried via {@link #getTransformMatrix(float[])}.  The transform matrix may change each
46  * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image
47  * is updated.
48  * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s,
49  * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in
50  * the streamed texture.  This transform compensates for any properties of the image stream source
51  * that cause it to appear different from a traditional OpenGL ES texture.  For example, sampling
52  * from the bottom left corner of the image can be accomplished by transforming the column vector
53  * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can
54  * be done by transforming (1, 1, 0, 1).
55  *
56  * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the
57  * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt">
58  * GL_OES_EGL_image_external</a> OpenGL ES extension.  This limits how the texture may be used.
59  * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than
60  * the GL_TEXTURE_2D target.  Additionally, any OpenGL ES 2.0 shader that samples from the texture
61  * must declare its use of this extension using, for example, an "#extension
62  * GL_OES_EGL_image_external : require" directive.  Such shaders must also access the texture using
63  * the samplerExternalOES GLSL sampler type.
64  *
65  * <p>SurfaceTexture objects may be created on any thread.  {@link #updateTexImage} may only be
66  * called on the thread with the OpenGL ES context that contains the texture object.  The
67  * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link
68  * #updateTexImage} should not be called directly from the callback.
69  */
70 public class SurfaceTexture {
71     private final Looper mCreatorLooper;
72     @UnsupportedAppUsage
73     private Handler mOnFrameAvailableHandler;
74 
75     /**
76      * These fields are used by native code, do not access or modify.
77      */
78     @UnsupportedAppUsage
79     private long mSurfaceTexture;
80     @UnsupportedAppUsage
81     private long mProducer;
82     @UnsupportedAppUsage
83     private long mFrameAvailableListener;
84 
85     private boolean mIsSingleBuffered;
86 
87     /**
88      * Callback interface for being notified that a new stream frame is available.
89      */
90     public interface OnFrameAvailableListener {
onFrameAvailable(SurfaceTexture surfaceTexture)91         void onFrameAvailable(SurfaceTexture surfaceTexture);
92     }
93 
94     /**
95      * Exception thrown when a SurfaceTexture couldn't be created or resized.
96      *
97      * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException}
98      * is used instead.
99      */
100     @SuppressWarnings("serial")
101     @Deprecated
102     public static class OutOfResourcesException extends Exception {
OutOfResourcesException()103         public OutOfResourcesException() {
104         }
OutOfResourcesException(String name)105         public OutOfResourcesException(String name) {
106             super(name);
107         }
108     }
109 
110     /**
111      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
112      *
113      * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
114      *
115      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
116      */
SurfaceTexture(int texName)117     public SurfaceTexture(int texName) {
118         this(texName, false);
119     }
120 
121     /**
122      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
123      * <p>
124      * In single buffered mode the application is responsible for serializing access to the image
125      * content buffer. Each time the image content is to be updated, the
126      * {@link #releaseTexImage()} method must be called before the image content producer takes
127      * ownership of the buffer. For example, when producing image content with the NDK
128      * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
129      * must be called before each ANativeWindow_lock, or that call will fail. When producing
130      * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
131      * OpenGL ES function call each frame.
132      *
133      * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
134      * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
135      *
136      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
137      */
SurfaceTexture(int texName, boolean singleBufferMode)138     public SurfaceTexture(int texName, boolean singleBufferMode) {
139         mCreatorLooper = Looper.myLooper();
140         mIsSingleBuffered = singleBufferMode;
141         nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this));
142     }
143 
144     /**
145      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
146      * <p>
147      * In single buffered mode the application is responsible for serializing access to the image
148      * content buffer. Each time the image content is to be updated, the
149      * {@link #releaseTexImage()} method must be called before the image content producer takes
150      * ownership of the buffer. For example, when producing image content with the NDK
151      * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
152      * must be called before each ANativeWindow_lock, or that call will fail. When producing
153      * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
154      * OpenGL ES function call each frame.
155      * <p>
156      * Unlike {@link #SurfaceTexture(int, boolean)}, which takes an OpenGL texture object name,
157      * this constructor creates the SurfaceTexture in detached mode. A texture name must be passed
158      * in using {@link #attachToGLContext} before calling {@link #releaseTexImage()} and producing
159      * image content using OpenGL ES.
160      *
161      * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
162      *
163      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
164      */
SurfaceTexture(boolean singleBufferMode)165     public SurfaceTexture(boolean singleBufferMode) {
166         mCreatorLooper = Looper.myLooper();
167         mIsSingleBuffered = singleBufferMode;
168         nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this));
169     }
170 
171     /**
172      * Register a callback to be invoked when a new image frame becomes available to the
173      * SurfaceTexture.
174      * <p>
175      * The callback may be called on an arbitrary thread, so it is not
176      * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the
177      * thread invoking the callback.
178      * </p>
179      *
180      * @param listener The listener to use, or null to remove the listener.
181      */
setOnFrameAvailableListener(@ullable OnFrameAvailableListener listener)182     public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) {
183         setOnFrameAvailableListener(listener, null);
184     }
185 
186     /**
187      * Register a callback to be invoked when a new image frame becomes available to the
188      * SurfaceTexture.
189      * <p>
190      * If a handler is specified, the callback will be invoked on that handler's thread.
191      * If no handler is specified, then the callback may be called on an arbitrary thread,
192      * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES
193      * context to the thread invoking the callback.
194      * </p>
195      *
196      * @param listener The listener to use, or null to remove the listener.
197      * @param handler The handler on which the listener should be invoked, or null
198      * to use an arbitrary thread.
199      */
setOnFrameAvailableListener(@ullable final OnFrameAvailableListener listener, @Nullable Handler handler)200     public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener,
201             @Nullable Handler handler) {
202         if (listener != null) {
203             // Although we claim the thread is arbitrary, earlier implementation would
204             // prefer to send the callback on the creating looper or the main looper
205             // so we preserve this behavior here.
206             Looper looper = handler != null ? handler.getLooper() :
207                     mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper();
208             mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) {
209                 @Override
210                 public void handleMessage(Message msg) {
211                     listener.onFrameAvailable(SurfaceTexture.this);
212                 }
213             };
214         } else {
215             mOnFrameAvailableHandler = null;
216         }
217     }
218 
219     /**
220      * Set the default size of the image buffers.  The image producer may override the buffer size,
221      * in which case the producer-set buffer size will be used, not the default size set by this
222      * method.  Both video and camera based image producers do override the size.  This method may
223      * be used to set the image size when producing images with {@link android.graphics.Canvas} (via
224      * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface).
225      * <p>
226      * The new default buffer size will take effect the next time the image producer requests a
227      * buffer to fill.  For {@link android.graphics.Canvas} this will be the next time {@link
228      * android.view.Surface#lockCanvas} is called.  For OpenGL ES, the EGLSurface should be
229      * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated
230      * (via {@code eglCreateWindowSurface}) to ensure that the new default size has taken effect.
231      * <p>
232      * The width and height parameters must be no greater than the minimum of
233      * {@code GL_MAX_VIEWPORT_DIMS} and {@code GL_MAX_TEXTURE_SIZE} (see
234      * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}).
235      * An error due to invalid dimensions might not be reported until
236      * updateTexImage() is called.
237      */
setDefaultBufferSize(int width, int height)238     public void setDefaultBufferSize(int width, int height) {
239         nativeSetDefaultBufferSize(width, height);
240     }
241 
242     /**
243      * Update the texture image to the most recent frame from the image stream.  This may only be
244      * called while the OpenGL ES context that owns the texture is current on the calling thread.
245      * It will implicitly bind its texture to the {@code GL_TEXTURE_EXTERNAL_OES} texture target.
246      */
updateTexImage()247     public void updateTexImage() {
248         nativeUpdateTexImage();
249     }
250 
251     /**
252      * Releases the the texture content. This is needed in single buffered mode to allow the image
253      * content producer to take ownership of the image buffer.
254      * <p>
255      * For more information see {@link #SurfaceTexture(int, boolean)}.
256      */
releaseTexImage()257     public void releaseTexImage() {
258         nativeReleaseTexImage();
259     }
260 
261     /**
262      * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object.
263      * This call must be made with the OpenGL ES context current on the calling thread.  The OpenGL
264      * ES texture object will be deleted as a result of this call.  After calling this method all
265      * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until
266      * a successful call to {@link #attachToGLContext} is made.
267      * <p>
268      * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
269      * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
270      * context at a time.
271      */
detachFromGLContext()272     public void detachFromGLContext() {
273         int err = nativeDetachFromGLContext();
274         if (err != 0) {
275             throw new RuntimeException("Error during detachFromGLContext (see logcat for details)");
276         }
277     }
278 
279     /**
280      * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread.  A
281      * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame
282      * that was current at the time of the last call to {@link #detachFromGLContext}.  This new
283      * texture is bound to the {@code GL_TEXTURE_EXTERNAL_OES} texture target.
284      * <p>
285      * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
286      * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
287      * context at a time.
288      *
289      * @param texName The name of the OpenGL ES texture that will be created.  This texture name
290      * must be unusued in the OpenGL ES context that is current on the calling thread.
291      */
attachToGLContext(int texName)292     public void attachToGLContext(int texName) {
293         int err = nativeAttachToGLContext(texName);
294         if (err != 0) {
295             throw new RuntimeException("Error during attachToGLContext (see logcat for details)");
296         }
297     }
298 
299     /**
300      * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by
301      * the most recent call to {@link #updateTexImage}.
302      * <p>
303      * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s
304      * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample
305      * that location from the texture.  Sampling the texture outside of the range of this transform
306      * is undefined.
307      * <p>
308      * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via
309      * the {@code glLoadMatrixf} or {@code glUniformMatrix4fv} functions.
310      * <p>
311      * If the underlying buffer has a crop associated with it, the transformation will also include
312      * a slight scale to cut off a 1-texel border around the edge of the crop. This ensures that
313      * when the texture is bilinear sampled that no texels outside of the buffer's valid region
314      * are accessed by the GPU, avoiding any sampling artifacts when scaling.
315      *
316      * @param mtx the array into which the 4x4 matrix will be stored.  The array must have exactly
317      *     16 elements.
318      */
getTransformMatrix(float[] mtx)319     public void getTransformMatrix(float[] mtx) {
320         // Note we intentionally don't check mtx for null, so this will result in a
321         // NullPointerException. But it's safe because it happens before the call to native.
322         if (mtx.length != 16) {
323             throw new IllegalArgumentException();
324         }
325         nativeGetTransformMatrix(mtx);
326     }
327 
328     /**
329      * Retrieve the timestamp associated with the texture image set by the most recent call to
330      * {@link #updateTexImage}.
331      *
332      * <p>This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp
333      * should be unaffected by time-of-day adjustments. The specific meaning and zero point of the
334      * timestamp depends on the source providing images to the SurfaceTexture. Unless otherwise
335      * specified by the image source, timestamps cannot generally be compared across SurfaceTexture
336      * instances, or across multiple program invocations. It is mostly useful for determining time
337      * offsets between subsequent frames.</p>
338      *
339      * <p>For camera sources, timestamps should be strictly monotonic. Timestamps from MediaPlayer
340      * sources may be reset when the playback position is set. For EGL and Vulkan producers, the
341      * timestamp is the desired present time set with the {@code EGL_ANDROID_presentation_time} or
342      * {@code VK_GOOGLE_display_timing} extensions.</p>
343      */
344 
getTimestamp()345     public long getTimestamp() {
346         return nativeGetTimestamp();
347     }
348 
349     /**
350      * {@code release()} frees all the buffers and puts the SurfaceTexture into the
351      * 'abandoned' state. Once put in this state the SurfaceTexture can never
352      * leave it. When in the 'abandoned' state, all methods of the
353      * {@code IGraphicBufferProducer} interface will fail with the {@code NO_INIT}
354      * error.
355      * <p>
356      * Note that while calling this method causes all the buffers to be freed
357      * from the perspective of the the SurfaceTexture, if there are additional
358      * references on the buffers (e.g. if a buffer is referenced by a client or
359      * by OpenGL ES as a texture) then those buffer will remain allocated.
360      * <p>
361      * Always call this method when you are done with SurfaceTexture. Failing
362      * to do so may delay resource deallocation for a significant amount of
363      * time.
364      *
365      * @see #isReleased()
366      */
release()367     public void release() {
368         nativeRelease();
369     }
370 
371     /**
372      * Returns {@code true} if the SurfaceTexture was released.
373      *
374      * @see #release()
375      */
isReleased()376     public boolean isReleased() {
377         return nativeIsReleased();
378     }
379 
380     @Override
finalize()381     protected void finalize() throws Throwable {
382         try {
383             nativeFinalize();
384         } finally {
385             super.finalize();
386         }
387     }
388 
389     /**
390      * This method is invoked from native code only.
391      */
392     @SuppressWarnings({"UnusedDeclaration"})
393     @UnsupportedAppUsage
postEventFromNative(WeakReference<SurfaceTexture> weakSelf)394     private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) {
395         SurfaceTexture st = weakSelf.get();
396         if (st != null) {
397             Handler handler = st.mOnFrameAvailableHandler;
398             if (handler != null) {
399                 handler.sendEmptyMessage(0);
400             }
401         }
402     }
403 
404     /**
405      * Returns {@code true} if the SurfaceTexture is single-buffered.
406      * @hide
407      */
isSingleBuffered()408     public boolean isSingleBuffered() {
409         return mIsSingleBuffered;
410     }
411 
nativeInit(boolean isDetached, int texName, boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)412     private native void nativeInit(boolean isDetached, int texName,
413             boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)
414             throws Surface.OutOfResourcesException;
nativeFinalize()415     private native void nativeFinalize();
nativeGetTransformMatrix(float[] mtx)416     private native void nativeGetTransformMatrix(float[] mtx);
nativeGetTimestamp()417     private native long nativeGetTimestamp();
nativeSetDefaultBufferSize(int width, int height)418     private native void nativeSetDefaultBufferSize(int width, int height);
nativeUpdateTexImage()419     private native void nativeUpdateTexImage();
nativeReleaseTexImage()420     private native void nativeReleaseTexImage();
421     @UnsupportedAppUsage
nativeDetachFromGLContext()422     private native int nativeDetachFromGLContext();
nativeAttachToGLContext(int texName)423     private native int nativeAttachToGLContext(int texName);
nativeRelease()424     private native void nativeRelease();
nativeIsReleased()425     private native boolean nativeIsReleased();
426 }
427