1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.filterpacks.videosrc;
18 
19 import android.filterfw.core.Filter;
20 import android.filterfw.core.FilterContext;
21 import android.filterfw.core.Frame;
22 import android.filterfw.core.FrameFormat;
23 import android.filterfw.core.GenerateFieldPort;
24 import android.filterfw.core.GenerateFinalPort;
25 import android.filterfw.core.GLFrame;
26 import android.filterfw.core.MutableFrameFormat;
27 import android.filterfw.core.ShaderProgram;
28 import android.filterfw.format.ImageFormat;
29 import android.graphics.SurfaceTexture;
30 import android.os.ConditionVariable;
31 import android.opengl.Matrix;
32 
33 import android.util.Log;
34 
35 /** <p>A filter that converts textures from a SurfaceTexture object into frames for
36  * processing in the filter framework.</p>
37  *
38  * <p>To use, connect up the sourceListener callback, and then when executing
39  * the graph, use the SurfaceTexture object passed to the callback to feed
40  * frames into the filter graph. For example, pass the SurfaceTexture into
41  * {#link
42  * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
43  * This filter is intended for applications that need for flexibility than the
44  * CameraSource and MediaSource provide. Note that the application needs to
45  * provide width and height information for the SurfaceTextureSource, which it
46  * should obtain from wherever the SurfaceTexture data is coming from to avoid
47  * unnecessary resampling.</p>
48  *
49  * @hide
50  */
51 public class SurfaceTextureSource extends Filter {
52 
53     /** User-visible parameters */
54 
55     /** The callback interface for the sourceListener parameter */
56     public interface SurfaceTextureSourceListener {
onSurfaceTextureSourceReady(SurfaceTexture source)57         public void onSurfaceTextureSourceReady(SurfaceTexture source);
58     }
59     /** A callback to send the internal SurfaceTexture object to, once it is
60      * created. This callback will be called when the the filter graph is
61      * preparing to execute, but before any processing has actually taken
62      * place. The SurfaceTexture object passed to this callback is the only way
63      * to feed this filter. When the filter graph is shutting down, this
64      * callback will be called again with null as the source.
65      *
66      * This callback may be called from an arbitrary thread, so it should not
67      * assume it is running in the UI thread in particular.
68      */
69     @GenerateFinalPort(name = "sourceListener")
70     private SurfaceTextureSourceListener mSourceListener;
71 
72     /** The width of the output image frame. If the texture width for the
73      * SurfaceTexture source is known, use it here to minimize resampling. */
74     @GenerateFieldPort(name = "width")
75     private int mWidth;
76 
77     /** The height of the output image frame. If the texture height for the
78      * SurfaceTexture source is known, use it here to minimize resampling. */
79     @GenerateFieldPort(name = "height")
80     private int mHeight;
81 
82     /** Whether the filter will always wait for a new frame from its
83      * SurfaceTexture, or whether it will output an old frame again if a new
84      * frame isn't available. The filter will always wait for the first frame,
85      * to avoid outputting a blank frame. Defaults to true.
86      */
87     @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
88     private boolean mWaitForNewFrame = true;
89 
90     /** Maximum timeout before signaling error when waiting for a new frame. Set
91      * this to zero to disable the timeout and wait indefinitely. In milliseconds.
92      */
93     @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
94     private int mWaitTimeout = 1000;
95 
96     /** Whether a timeout is an exception-causing failure, or just causes the
97      * filter to close.
98      */
99     @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
100     private boolean mCloseOnTimeout = false;
101 
102     // Variables for input->output conversion
103     private GLFrame mMediaFrame;
104     private ShaderProgram mFrameExtractor;
105     private SurfaceTexture mSurfaceTexture;
106     private MutableFrameFormat mOutputFormat;
107     private ConditionVariable mNewFrameAvailable;
108     private boolean mFirstFrame;
109 
110     private float[] mFrameTransform;
111     private float[] mMappedCoords;
112     // These default source coordinates perform the necessary flip
113     // for converting from MFF/Bitmap origin to OpenGL origin.
114     private static final float[] mSourceCoords = { 0, 1, 0, 1,
115                                                    1, 1, 0, 1,
116                                                    0, 0, 0, 1,
117                                                    1, 0, 0, 1 };
118     // Shader for output
119     private final String mRenderShader =
120             "#extension GL_OES_EGL_image_external : require\n" +
121             "precision mediump float;\n" +
122             "uniform samplerExternalOES tex_sampler_0;\n" +
123             "varying vec2 v_texcoord;\n" +
124             "void main() {\n" +
125             "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
126             "}\n";
127 
128     // Variables for logging
129 
130     private static final String TAG = "SurfaceTextureSource";
131     private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
132 
SurfaceTextureSource(String name)133     public SurfaceTextureSource(String name) {
134         super(name);
135         mNewFrameAvailable = new ConditionVariable();
136         mFrameTransform = new float[16];
137         mMappedCoords = new float[16];
138     }
139 
140     @Override
setupPorts()141     public void setupPorts() {
142         // Add input port
143         addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
144                                                   FrameFormat.TARGET_GPU));
145     }
146 
createFormats()147     private void createFormats() {
148         mOutputFormat = ImageFormat.create(mWidth, mHeight,
149                                            ImageFormat.COLORSPACE_RGBA,
150                                            FrameFormat.TARGET_GPU);
151     }
152 
153     @Override
prepare(FilterContext context)154     protected void prepare(FilterContext context) {
155         if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");
156 
157         createFormats();
158 
159         // Prepare input
160         mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
161                                                                        GLFrame.EXTERNAL_TEXTURE,
162                                                                        0);
163 
164         // Prepare output
165         mFrameExtractor = new ShaderProgram(context, mRenderShader);
166     }
167 
168     @Override
open(FilterContext context)169     public void open(FilterContext context) {
170         if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
171         // Create SurfaceTexture anew each time - it can use substantial memory.
172         mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
173         // Connect SurfaceTexture to callback
174         mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
175         // Connect SurfaceTexture to source
176         mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
177         mFirstFrame = true;
178     }
179 
180     @Override
process(FilterContext context)181     public void process(FilterContext context) {
182         if (mLogVerbose) Log.v(TAG, "Processing new frame");
183 
184         // First, get new frame if available
185         if (mWaitForNewFrame || mFirstFrame) {
186             boolean gotNewFrame;
187             if (mWaitTimeout != 0) {
188                 gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
189                 if (!gotNewFrame) {
190                     if (!mCloseOnTimeout) {
191                         throw new RuntimeException("Timeout waiting for new frame");
192                     } else {
193                         if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
194                         closeOutputPort("video");
195                         return;
196                     }
197                 }
198             } else {
199                 mNewFrameAvailable.block();
200             }
201             mNewFrameAvailable.close();
202             mFirstFrame = false;
203         }
204 
205         mSurfaceTexture.updateTexImage();
206 
207         mSurfaceTexture.getTransformMatrix(mFrameTransform);
208         Matrix.multiplyMM(mMappedCoords, 0,
209                           mFrameTransform, 0,
210                           mSourceCoords, 0);
211         mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
212                                         mMappedCoords[4], mMappedCoords[5],
213                                         mMappedCoords[8], mMappedCoords[9],
214                                         mMappedCoords[12], mMappedCoords[13]);
215         // Next, render to output
216         Frame output = context.getFrameManager().newFrame(mOutputFormat);
217         mFrameExtractor.process(mMediaFrame, output);
218 
219         output.setTimestamp(mSurfaceTexture.getTimestamp());
220 
221         pushOutput("video", output);
222         output.release();
223     }
224 
225     @Override
close(FilterContext context)226     public void close(FilterContext context) {
227         if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
228         mSourceListener.onSurfaceTextureSourceReady(null);
229         mSurfaceTexture.release();
230         mSurfaceTexture = null;
231     }
232 
233     @Override
tearDown(FilterContext context)234     public void tearDown(FilterContext context) {
235         if (mMediaFrame != null) {
236             mMediaFrame.release();
237         }
238     }
239 
240     @Override
fieldPortValueUpdated(String name, FilterContext context)241     public void fieldPortValueUpdated(String name, FilterContext context) {
242         if (name.equals("width") || name.equals("height") ) {
243             mOutputFormat.setDimensions(mWidth, mHeight);
244         }
245     }
246 
247     private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
248             new SurfaceTexture.OnFrameAvailableListener() {
249         public void onFrameAvailable(SurfaceTexture surfaceTexture) {
250             if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
251             mNewFrameAvailable.open();
252         }
253     };
254 }
255