1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.example.nativemedia;
18 
19 import android.graphics.SurfaceTexture;
20 import android.util.Log;
21 
22 import java.nio.ByteBuffer;
23 import java.nio.ByteOrder;
24 import java.nio.FloatBuffer;
25 
26 import javax.microedition.khronos.egl.EGLConfig;
27 import javax.microedition.khronos.opengles.GL10;
28 
29 import android.view.MotionEvent;
30 import android.content.Context;
31 
32 import android.opengl.GLES20;
33 import android.opengl.GLSurfaceView;
34 import android.opengl.Matrix;
35 
36 import android.hardware.SensorManager;
37 import android.hardware.SensorEvent;
38 import android.hardware.SensorEventListener;
39 import android.hardware.Sensor;
40 
41 // Remove once surfacetexture timestamps are in
42 import java.lang.System;
43 
44 import android.util.AttributeSet;
45 
46 public class MyGLSurfaceView extends GLSurfaceView {
47 
48     MyRenderer mRenderer;
49 
MyGLSurfaceView(Context context)50     public MyGLSurfaceView(Context context) {
51         super(context, null);
52     }
53 
MyGLSurfaceView(Context context, AttributeSet attributeSet)54     public MyGLSurfaceView(Context context, AttributeSet attributeSet) {
55         super(context, attributeSet);
56         init(context);
57     }
58 
init(Context context)59     private void init(Context context) {
60         setEGLContextClientVersion(2);
61         mRenderer = new MyRenderer(context);
62         setRenderer(mRenderer);
63     }
64 
65     @Override
onPause()66     public void onPause() {
67         super.onPause();
68     }
69 
70     @Override
onResume()71     public void onResume() {
72         super.onResume();
73     }
74 
getSurfaceTexture()75     public SurfaceTexture getSurfaceTexture() {
76         return mRenderer.getSurfaceTexture();
77     }
78 }
79 
80 class MyRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
81     private Context mContext;
82 
MyRenderer(Context context)83     public MyRenderer(Context context) {
84         mContext = context;
85 
86         mVertices = ByteBuffer.allocateDirect(mVerticesData.length
87                 * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
88         mVertices.put(mVerticesData).position(0);
89 
90         Matrix.setIdentityM(mSTMatrix, 0);
91         Matrix.setIdentityM(mMMatrix, 0);
92         Matrix.rotateM(mMMatrix, 0, 20, 0, 1, 0);
93     }
94 
onDrawFrame(GL10 glUnused)95     public void onDrawFrame(GL10 glUnused) {
96         synchronized(this) {
97             if (updateSurface) {
98                 mSurface.updateTexImage();
99 
100                 mSurface.getTransformMatrix(mSTMatrix);
101                 updateSurface = false;
102             }
103         }
104 
105         // Ignore the passed-in GL10 interface, and use the GLES20
106         // class's static methods instead.
107         GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
108         GLES20.glUseProgram(mProgram);
109         checkGlError("glUseProgram");
110 
111         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
112         GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
113 
114         mVertices.position(VERTICES_DATA_POS_OFFSET);
115         GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
116                 VERTICES_DATA_STRIDE_BYTES, mVertices);
117         checkGlError("glVertexAttribPointer maPosition");
118         GLES20.glEnableVertexAttribArray(maPositionHandle);
119         checkGlError("glEnableVertexAttribArray maPositionHandle");
120 
121         mVertices.position(VERTICES_DATA_UV_OFFSET);
122         GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
123                 VERTICES_DATA_STRIDE_BYTES, mVertices);
124         checkGlError("glVertexAttribPointer maTextureHandle");
125         GLES20.glEnableVertexAttribArray(maTextureHandle);
126         checkGlError("glEnableVertexAttribArray maTextureHandle");
127 
128         Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0);
129         Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0);
130 
131         GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
132         GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
133 
134         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
135         checkGlError("glDrawArrays");
136     }
137 
onSurfaceChanged(GL10 glUnused, int width, int height)138     public void onSurfaceChanged(GL10 glUnused, int width, int height) {
139         // Ignore the passed-in GL10 interface, and use the GLES20
140         // class's static methods instead.
141         GLES20.glViewport(0, 0, width, height);
142         mRatio = (float) width / height;
143         Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7);
144     }
145 
onSurfaceCreated(GL10 glUnused, EGLConfig config)146     public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
147         // Ignore the passed-in GL10 interface, and use the GLES20
148         // class's static methods instead.
149 
150         /* Set up alpha blending and an Android background color */
151         GLES20.glEnable(GLES20.GL_BLEND);
152         GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
153         GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f);
154 
155         /* Set up shaders and handles to their variables */
156         mProgram = createProgram(mVertexShader, mFragmentShader);
157         if (mProgram == 0) {
158             return;
159         }
160         maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
161         checkGlError("glGetAttribLocation aPosition");
162         if (maPositionHandle == -1) {
163             throw new RuntimeException("Could not get attrib location for aPosition");
164         }
165         maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
166         checkGlError("glGetAttribLocation aTextureCoord");
167         if (maTextureHandle == -1) {
168             throw new RuntimeException("Could not get attrib location for aTextureCoord");
169         }
170 
171         muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
172         checkGlError("glGetUniformLocation uMVPMatrix");
173         if (muMVPMatrixHandle == -1) {
174             throw new RuntimeException("Could not get attrib location for uMVPMatrix");
175         }
176 
177         muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
178         checkGlError("glGetUniformLocation uSTMatrix");
179         if (muMVPMatrixHandle == -1) {
180             throw new RuntimeException("Could not get attrib location for uSTMatrix");
181         }
182 
183         checkGlError("glGetUniformLocation uCRatio");
184         if (muMVPMatrixHandle == -1) {
185             throw new RuntimeException("Could not get attrib location for uCRatio");
186         }
187 
188         /*
189          * Create our texture. This has to be done each time the
190          * surface is created.
191          */
192 
193         int[] textures = new int[1];
194         GLES20.glGenTextures(1, textures, 0);
195 
196         mTextureID = textures[0];
197         GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
198         checkGlError("glBindTexture mTextureID");
199 
200         // Can't do mipmapping with camera source
201         GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
202                 GLES20.GL_NEAREST);
203         GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
204                 GLES20.GL_LINEAR);
205         // Clamp to edge is the only option
206         GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
207                 GLES20.GL_CLAMP_TO_EDGE);
208         GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
209                 GLES20.GL_CLAMP_TO_EDGE);
210         checkGlError("glTexParameteri mTextureID");
211 
212         /*
213          * Create the SurfaceTexture that will feed this textureID, and pass it to the camera
214          */
215 
216         mSurface = new SurfaceTexture(mTextureID);
217         mSurface.setOnFrameAvailableListener(this);
218 
219         Matrix.setLookAtM(mVMatrix, 0, 0, 0, 4f, 0f, 0f, 0f, 0f, 1.0f, 0.0f);
220 
221         synchronized(this) {
222             updateSurface = false;
223         }
224     }
225 
onFrameAvailable(SurfaceTexture surface)226     synchronized public void onFrameAvailable(SurfaceTexture surface) {
227         /* For simplicity, SurfaceTexture calls here when it has new
228          * data available.  Call may come in from some random thread,
229          * so let's be safe and use synchronize. No OpenGL calls can be done here.
230          */
231         updateSurface = true;
232         //Log.v(TAG, "onFrameAvailable " + surface.getTimestamp());
233     }
234 
loadShader(int shaderType, String source)235     private int loadShader(int shaderType, String source) {
236         int shader = GLES20.glCreateShader(shaderType);
237         if (shader != 0) {
238             GLES20.glShaderSource(shader, source);
239             GLES20.glCompileShader(shader);
240             int[] compiled = new int[1];
241             GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
242             if (compiled[0] == 0) {
243                 Log.e(TAG, "Could not compile shader " + shaderType + ":");
244                 Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
245                 GLES20.glDeleteShader(shader);
246                 shader = 0;
247             }
248         }
249         return shader;
250     }
251 
createProgram(String vertexSource, String fragmentSource)252     private int createProgram(String vertexSource, String fragmentSource) {
253         int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
254         if (vertexShader == 0) {
255             return 0;
256         }
257         int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
258         if (pixelShader == 0) {
259             return 0;
260         }
261 
262         int program = GLES20.glCreateProgram();
263         if (program != 0) {
264             GLES20.glAttachShader(program, vertexShader);
265             checkGlError("glAttachShader");
266             GLES20.glAttachShader(program, pixelShader);
267             checkGlError("glAttachShader");
268             GLES20.glLinkProgram(program);
269             int[] linkStatus = new int[1];
270             GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
271             if (linkStatus[0] != GLES20.GL_TRUE) {
272                 Log.e(TAG, "Could not link program: ");
273                 Log.e(TAG, GLES20.glGetProgramInfoLog(program));
274                 GLES20.glDeleteProgram(program);
275                 program = 0;
276             }
277         }
278         return program;
279     }
280 
checkGlError(String op)281     private void checkGlError(String op) {
282         int error;
283         while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
284             Log.e(TAG, op + ": glError " + error);
285             throw new RuntimeException(op + ": glError " + error);
286         }
287     }
288 
289     private static final int FLOAT_SIZE_BYTES = 4;
290     private static final int VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
291     private static final int VERTICES_DATA_POS_OFFSET = 0;
292     private static final int VERTICES_DATA_UV_OFFSET = 3;
293     private final float[] mVerticesData = {
294         // X, Y, Z, U, V
295         -1.0f, -1.0f, 0, 0.f, 0.f,
296         1.0f, -1.0f, 0, 1.f, 0.f,
297         -1.0f,  1.0f, 0, 0.f, 1.f,
298         1.0f,   1.0f, 0, 1.f, 1.f,
299     };
300 
301     private FloatBuffer mVertices;
302 
303     private final String mVertexShader =
304         "uniform mat4 uMVPMatrix;\n" +
305         "uniform mat4 uSTMatrix;\n" +
306         "attribute vec4 aPosition;\n" +
307         "attribute vec4 aTextureCoord;\n" +
308         "varying vec2 vTextureCoord;\n" +
309         "void main() {\n" +
310         "  gl_Position = uMVPMatrix * aPosition;\n" +
311         "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
312         "}\n";
313 
314     private final String mFragmentShader =
315         "#extension GL_OES_EGL_image_external : require\n" +
316         "precision mediump float;\n" +
317         "varying vec2 vTextureCoord;\n" +
318         "uniform samplerExternalOES sTexture;\n" +
319         "void main() {\n" +
320         "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
321         "}\n";
322 
323     private float[] mMVPMatrix = new float[16];
324     private float[] mProjMatrix = new float[16];
325     private float[] mMMatrix = new float[16];
326     private float[] mVMatrix = new float[16];
327     private float[] mSTMatrix = new float[16];
328 
329     private int mProgram;
330     private int mTextureID;
331     private int muMVPMatrixHandle;
332     private int muSTMatrixHandle;
333     private int maPositionHandle;
334     private int maTextureHandle;
335 
336     private float mRatio = 1.0f;
337     private SurfaceTexture mSurface;
338     private boolean updateSurface = false;
339 
340     private static final String TAG = "MyRenderer";
341 
342     // Magic key
343     private static final int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
344 
getSurfaceTexture()345     public SurfaceTexture getSurfaceTexture() {
346         return mSurface;
347     }
348 }
349