Android OpenGL 以原始（小）分辨率渲染到 frameBuffer，然后将其缩放（viewPort）到屏幕尺寸
Android OpenGL Render to a frameBuffer in the original (small) resolution and then scale it (viewPort) to the screen size
我有一个 NV21 (YUV420) 摄像机视频，我正在对其应用片段着色器以获得一些滤镜效果以及 YUV 到 RGB 的转换。
除了性能不佳外，一切正常。
我的片段着色器有点重，因为它有很多 textur2D() 调用。
原始帧分辨率为 480x640 像素，我注意到如果我将视口设置为原始大小（而不是全屏大小），它运行良好且流畅。
所以基本上我需要首先渲染帧并在具有原始大小的 frameBuffer (FBO) 中处理它，然后（在着色器工作完成后）使用视口（主要是 1080x1920）将它缩放到全屏大小，并且这意味着 "heavy" 处理工作将应用于更少的片段。
我在这里找到了一些教程和类似的问题如何实现，但不幸的是我没有运气。 （有一些黑屏或 GL_INVALID_OPERATION 等'）...
任何帮助将不胜感激。
此外，我不知道如何处理（如果可能）的另一个（可选）性能调整是以某种方式组合这 3 个纹理（Y_tex、U_tex 和 V_tex) 到单个纹理，作为单个采样器统一到着色器，然后我可以在着色器中进行一次 texture2D() 调用，以获取当前 YUV 值并将它们转换为 RGB 值。
这是我的渲染器代码：
static class MyRenderer implements GLSurfaceView.Renderer
{
int mTextureIds[] = new int[3];
float[] mScaleMatrix = new float[16];
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private ShortBuffer mDrawListBuffer;
boolean mVideoFitEnabled = true;
boolean mVideoDisabled = false;
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static final int TEXTURECOORDS_PER_VERTEX = 2;
static float mXYZCoords[] = {
-1.0f, 1.0f, 0.0f, // top left
-1.0f, -1.0f, 0.0f, // bottom left
1.0f, -1.0f, 0.0f, // bottom right
1.0f, 1.0f, 0.0f // top right
};
static float mUVCoords[] = {
0, 0, // top left
0, 1, // bottom left
1, 1, // bottom right
1, 0 // top right
};
private short mVertexIndex[] = {0, 1, 2, 0, 2, 3}; // order to draw vertices
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;"
+ "attribute vec4 aPosition;\n"
+ "attribute vec2 aTextureCoord;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main() {\n"
+ " gl_Position = uMVPMatrix * aPosition;\n"
+ " vTextureCoord = aTextureCoord;\n"
+ "}\n";
private final String fragmentShaderCode =
"precision mediump float;\n"
+ "uniform sampler2D Ytex;\n"
+ "uniform sampler2D Utex,Vtex;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main(void) {\n"
+ " float nx,ny,r,g,b,y,u,v;\n"
+ " mediump vec4 txl,ux,vx;"
+ " nx=vTextureCoord[0];\n"
+ " ny=vTextureCoord[1];\n"
+ " y=texture2D(Ytex,vec2(nx,ny)).r;\n"
+ " u=texture2D(Utex,vec2(nx,ny)).r;\n"
+ " v=texture2D(Vtex,vec2(nx,ny)).r;\n"
+ " y=1.1643*(y-0.0625);\n"
+ " u=u-0.5;\n"
+ " v=v-0.5;\n"
+ " r=y+1.5958*v;\n"
+ " g=y-0.39173*u-0.81290*v;\n"
+ " b=y+2.017*u;\n"
// --> Bilateral blur filter code HERE <--
+ " gl_FragColor=vec4(r,g,b,1.0);\n"
+ "}\n";
ReentrantLock mFrameLock = new ReentrantLock();
Frame mCurrentFrame;
private int mProgram;
private int mTextureWidth;
private int mTextureHeight;
private int mViewportWidth;
private int mViewportHeight;
public MyRenderer()
{
ByteBuffer bb = ByteBuffer.allocateDirect(mXYZCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mXYZCoords);
mVertexBuffer.position(0);
ByteBuffer tb = ByteBuffer.allocateDirect(mUVCoords.length * 4);
tb.order(ByteOrder.nativeOrder());
mTextureBuffer = tb.asFloatBuffer();
mTextureBuffer.put(mUVCoords);
mTextureBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(mVertexIndex.length * 2);
dlb.order(ByteOrder.nativeOrder());
mDrawListBuffer = dlb.asShortBuffer();
mDrawListBuffer.put(mVertexIndex);
mDrawListBuffer.position(0);
}
@Override public void onSurfaceCreated(GL10 gl, EGLConfig config)
{
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram);
int positionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
int textureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4, mVertexBuffer);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(textureHandle, TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false, TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer);
GLES20.glEnableVertexAttribArray(textureHandle);
GLES20.glUseProgram(mProgram);
int i = GLES20.glGetUniformLocation(mProgram, "Ytex"); // GLES20.glUniform3i(i, 0, 1, 2);
GLES20.glUniform1i(i, 0); /* Bind Ytex to texture unit 0 */
i = GLES20.glGetUniformLocation(mProgram, "Utex");
GLES20.glUniform1i(i, 1); /* Bind Utex to texture unit 1 */
i = GLES20.glGetUniformLocation(mProgram, "Vtex");
GLES20.glUniform1i(i, 2); /* Bind Vtex to texture unit 2 */
mTextureWidth = 0;
mTextureHeight = 0;
}
static void initializeTexture(int name, int id, int width, int height)
{
GLES20.glActiveTexture(name);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width, height, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
}
void setupTextures(Frame frame)
{
if (mTextureIds[0] != 0)
{
GLES20.glDeleteTextures(3, mTextureIds, 0);
}
GLES20.glGenTextures(3, mTextureIds, 0);
int w = frame.getWidth();
int h = frame.getHeight();
int hw = (w + 1) >> 1;
int hh = (h + 1) >> 1;
initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h);
initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh);
initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh);
mTextureWidth = frame.getWidth();
mTextureHeight = frame.getHeight();
}
void updateTextures(Frame frame)
{
int width = frame.getWidth();
int height = frame.getHeight();
int half_width = (width + 1) >> 1;
int half_height = (height + 1) >> 1;
int y_size = width * height;
int uv_size = half_width * half_height;
ByteBuffer bb = frame.getBuffer();
bb.clear(); // If we are reusing this frame, make sure we reset position and limit
if (bb.remaining() == y_size + uv_size * 2)
{
bb.position(0);
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size + uv_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
int i = GLES20.glGetUniformLocation(mProgram, "width");
GLES20.glUniform1f(i, (float) mTextureWidth);
i = GLES20.glGetUniformLocation(mProgram, "height");
GLES20.glUniform1f(i, (float) mTextureHeight);
}
else
{
mTextureWidth = 0;
mTextureHeight = 0;
}
}
@Override public void onSurfaceChanged(GL10 gl, int width, int height)
{
GLES20.glViewport(0, 0, width, height);
mViewportWidth = width;
mViewportHeight = height;
}
@Override public void onDrawFrame(GL10 gl)
{
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
mFrameLock.lock();
if (mCurrentFrame != null && !mVideoDisabled)
{
GLES20.glUseProgram(mProgram);
if (mTextureWidth != mCurrentFrame.getWidth() || mTextureHeight != mCurrentFrame.getHeight())
{
setupTextures(mCurrentFrame);
}
updateTextures(mCurrentFrame);
Matrix.setIdentityM(mScaleMatrix, 0);
float scaleX = 1.0f, scaleY = 1.0f;
float ratio = (float) mCurrentFrame.getWidth() / mCurrentFrame.getHeight();
float vratio = (float) mViewportWidth / mViewportHeight;
if (mVideoFitEnabled)
{
if (ratio > vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
else
{
if (ratio < vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
Matrix.scaleM(mScaleMatrix, 0, scaleX * (mCurrentFrame.isMirroredX() ? -1.0f : 1.0f), scaleY, 1);
int mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mScaleMatrix, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
}
mFrameLock.unlock();
}
public void displayFrame(Frame frame)
{
mFrameLock.lock();
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = frame;
mFrameLock.unlock();
}
public static int loadShader(int type, String shaderCode)
{
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
public void disableVideo(boolean b)
{
mFrameLock.lock();
mVideoDisabled = b;
if (mVideoDisabled)
{
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = null;
}
mFrameLock.unlock();
}
public void enableVideoFit(boolean enableVideoFit)
{
mVideoFitEnabled = enableVideoFit;
}
}
您要做的是 downsampling.What 您需要做的是首先渲染成一个小的 FBO，其中视口和 FBO 附件的大小相同。在该通道中，您可以应用模糊效果。然后您可以将其渲染或 blit 到另一个原始大小的 FBO 以获得缩放的纹理 back.It 重要的是要注意，根据您的蓝色技术，放大后的结果可能会有明显的质量下降。
由于某些 OpenGL API 调用在 Android SDK 中可能有不同的命名，这里是您需要执行的一般伪代码：
设 w = 原始宽度，h = 原始高度。
1.Create 带有大小为 w/2 和 h/2 的纹理附件的自定义 FBO（如果您计划以原始分辨率的一半进行下采样。
2.Attach纹理到FBO.BindFBO写入。
3.set glViewport() 的大小与 FBO 纹理附件相同。
4.Render 全屏 quad.Apply 你在本关中的效果。
5.Second pass:Bind 返回默认帧缓冲区（或者 - 如果您稍后有其他渲染阶段，则使用另一个自定义 FBO）。如果您只想 blit，请确保绑定第一个 FBO只读不写。
6.Bind 从上一次传递到采样器单元附加到 FBO 的纹理。
7.Set w 和 h 的视口 size.Draw 全屏 quad.Do 在片段着色器中无论你需要什么。（伽玛校正，混合等..）
it.What 在这种技术中的好处在于 OpenGL 会自动为您执行 upscale/downscale 过滤类型，您 select 用于处理过程中涉及的纹理。
另一方面，note:If 你有性能问题，在 Java.Though 中编写渲染器不是一个好主意，你的问题可能是由于错误的 API 用法或模糊效果的低效算法应用。
最终我弄明白了，感谢一位非常有才华的人帮助了我。
这是我的渲染器 class，它现在有一个带有 2 个渲染通道的 frameBuffer：
static class MyRenderer implements GLSurfaceView.Renderer
{
int mTextureIds[] = new int[4];
float[] mScaleMatrix = new float[16];
float[] mFilterScaleMatrix = new float[16];
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private ShortBuffer mDrawListBuffer;
private IntBuffer frameBuffer;
boolean mVideoFitEnabled = true;
boolean mVideoDisabled = false;
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static final int TEXTURECOORDS_PER_VERTEX = 2;
static float mXYZCoords[] = {
-1.0f, 1.0f, 0.0f, // top left
-1.0f, -1.0f, 0.0f, // bottom left
1.0f, -1.0f, 0.0f, // bottom right
1.0f, 1.0f, 0.0f // top right
};
static float mUVCoords[] = {
0, 0, // top left
0, 1, // bottom left
1, 1, // bottom right
1, 0 // top right
};
private short mVertexIndex[] = {0, 1, 2, 0, 2, 3}; // order to draw vertices
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;"
+ "attribute vec4 aPosition;\n"
+ "attribute vec2 aTextureCoord;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main() {\n"
+ " gl_Position = uMVPMatrix * aPosition;\n"
+ " vTextureCoord = aTextureCoord;\n"
+ "}\n";
private final String fragmentShaderCode =
"YUV to RGB Conversion shader HERE";
private final String frameBufferShader =
"MY filter effect shader HERE";
ReentrantLock mFrameLock = new ReentrantLock();
Frame mCurrentFrame;
private int mProgram;
private int mProgramFilter;
private int mTextureWidth;
private int mTextureHeight;
private int mViewportWidth;
private int mViewportHeight;
public MyRenderer()
{
ByteBuffer bb = ByteBuffer.allocateDirect(mXYZCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mXYZCoords);
mVertexBuffer.position(0);
ByteBuffer tb = ByteBuffer.allocateDirect(mUVCoords.length * 4);
tb.order(ByteOrder.nativeOrder());
mTextureBuffer = tb.asFloatBuffer();
mTextureBuffer.put(mUVCoords);
mTextureBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(mVertexIndex.length * 2);
dlb.order(ByteOrder.nativeOrder());
mDrawListBuffer = dlb.asShortBuffer();
mDrawListBuffer.put(mVertexIndex);
mDrawListBuffer.position(0);
frameBuffer = IntBuffer.allocate(1);
}
@Override public void onSurfaceCreated(GL10 gl, EGLConfig config)
{
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
int filterVertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int filterFragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, frameBufferShader);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram);
mProgramFilter = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgramFilter, filterVertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgramFilter, filterFragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgramFilter);
int positionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
int textureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4, mVertexBuffer);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(textureHandle, TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false, TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer);
GLES20.glEnableVertexAttribArray(textureHandle);
GLES20.glUseProgram(mProgram);
int i = GLES20.glGetUniformLocation(mProgram, "Ytex");
GLES20.glUniform1i(i, 3); /* Bind Ytex to texture unit 0 */
i = GLES20.glGetUniformLocation(mProgram, "Utex");
GLES20.glUniform1i(i, 1); /* Bind Utex to texture unit 1 */
i = GLES20.glGetUniformLocation(mProgram, "Vtex");
GLES20.glUniform1i(i, 2); /* Bind Vtex to texture unit 2 */
GLES20.glUseProgram(mProgramFilter);
i = GLES20.glGetUniformLocation(mProgramFilter, "Ytex");
GLES20.glUniform1i(i, 0);
mTextureWidth = 0;
mTextureHeight = 0;
}
static void initializeTexture(int name, int id, int width, int height)
{
GLES20.glActiveTexture(name);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width, height, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
}
void setupTextures(Frame frame)
{
if (mTextureIds[0] != 0)
{
GLES20.glDeleteTextures(4, mTextureIds, 0);
}
GLES20.glGenTextures(4, mTextureIds, 0);
int w = frame.getWidth();
int h = frame.getHeight();
int hw = (w + 1) >> 1;
int hh = (h + 1) >> 1;
initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h);
initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh);
initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh);
GLES20.glGenFramebuffers(1, frameBuffer);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer.get(0));
GLES20.glActiveTexture(GLES20.GL_TEXTURE3);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[3]);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, w, h, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, mTextureIds[3], 0);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
mTextureWidth = frame.getWidth();
mTextureHeight = frame.getHeight();
GLES20.glUseProgram(mProgramFilter);
int i = GLES20.glGetUniformLocation(mProgramFilter, "width");
GLES20.glUniform1f(i, (float) mTextureWidth);
i = GLES20.glGetUniformLocation(mProgramFilter, "height");
GLES20.glUniform1f(i, (float) mTextureHeight);
}
void updateTextures(Frame frame)
{
int width = frame.getWidth();
int height = frame.getHeight();
int half_width = (width + 1) >> 1;
int half_height = (height + 1) >> 1;
int y_size = width * height;
int uv_size = half_width * half_height;
ByteBuffer bb = frame.getBuffer();
bb.clear(); // If we are reusing this frame, make sure we reset position and limit
if (bb.remaining() == y_size + uv_size * 2)
{
bb.position(0);
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size + uv_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
}
else
{
mTextureWidth = 0;
mTextureHeight = 0;
}
}
@Override public void onSurfaceChanged(GL10 gl, int width, int height)
{
/// GLES20.glViewport(0, 0, width, height);
mViewportWidth = width;
mViewportHeight = height;
}
@Override public void onDrawFrame(GL10 gl)
{
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
mFrameLock.lock();
if (mCurrentFrame != null && !mVideoDisabled)
{
if (mTextureWidth != mCurrentFrame.getWidth() || mTextureHeight != mCurrentFrame.getHeight())
{
setupTextures(mCurrentFrame);
}
updateTextures(mCurrentFrame);
/// Step 1: Smoothing Filter - Render to FrameBuffer [pass 1]
Matrix.setIdentityM(mFilterScaleMatrix, 0);
GLES20.glViewport(0, 0, mTextureWidth, mTextureHeight);
GLES20.glUseProgram(mProgramFilter);
int mMVPFilterMatrixHandle = GLES20.glGetUniformLocation(mProgramFilter, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPFilterMatrixHandle, 1, false, mFilterScaleMatrix, 0);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer.get(0));
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
/// Step 2: Draw + RGB Conversion - Render to screen [pass 2]
Matrix.setIdentityM(mScaleMatrix, 0);
float scaleX = 1.0f, scaleY = 1.0f;
float ratio = (float) mCurrentFrame.getWidth() / mCurrentFrame.getHeight();
float vratio = (float) mViewportWidth / mViewportHeight;
if (mVideoFitEnabled)
{
if (ratio > vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
else
{
if (ratio < vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
Matrix.scaleM(mScaleMatrix, 0, scaleX * (mCurrentFrame.isMirroredX() ? -1.0f : 1.0f), scaleY, 1);
GLES20.glUseProgram(mProgram);
GLES20.glViewport(0, 0, mViewportWidth, mViewportHeight);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
int mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mScaleMatrix, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
}
mFrameLock.unlock();
}
public void displayFrame(Frame frame)
{
mFrameLock.lock();
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = frame;
mFrameLock.unlock();
}
public static int loadShader(int type, String shaderCode)
{
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
public void disableVideo(boolean b)
{
mFrameLock.lock();
mVideoDisabled = b;
if (mVideoDisabled)
{
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = null;
}
mFrameLock.unlock();
}
public void enableVideoFit(boolean enableVideoFit)
{
mVideoFitEnabled = enableVideoFit;
}
}
我有一个 NV21 (YUV420) 摄像机视频，我正在对其应用片段着色器以获得一些滤镜效果以及 YUV 到 RGB 的转换。
除了性能不佳外，一切正常。
我的片段着色器有点重，因为它有很多 textur2D() 调用。
原始帧分辨率为 480x640 像素，我注意到如果我将视口设置为原始大小（而不是全屏大小），它运行良好且流畅。
所以基本上我需要首先渲染帧并在具有原始大小的 frameBuffer (FBO) 中处理它，然后（在着色器工作完成后）使用视口（主要是 1080x1920）将它缩放到全屏大小，并且这意味着 "heavy" 处理工作将应用于更少的片段。
我在这里找到了一些教程和类似的问题如何实现，但不幸的是我没有运气。 （有一些黑屏或 GL_INVALID_OPERATION 等'）... 任何帮助将不胜感激。
此外，我不知道如何处理（如果可能）的另一个（可选）性能调整是以某种方式组合这 3 个纹理（Y_tex、U_tex 和 V_tex) 到单个纹理，作为单个采样器统一到着色器，然后我可以在着色器中进行一次 texture2D() 调用，以获取当前 YUV 值并将它们转换为 RGB 值。
这是我的渲染器代码：
static class MyRenderer implements GLSurfaceView.Renderer
{
int mTextureIds[] = new int[3];
float[] mScaleMatrix = new float[16];
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private ShortBuffer mDrawListBuffer;
boolean mVideoFitEnabled = true;
boolean mVideoDisabled = false;
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static final int TEXTURECOORDS_PER_VERTEX = 2;
static float mXYZCoords[] = {
-1.0f, 1.0f, 0.0f, // top left
-1.0f, -1.0f, 0.0f, // bottom left
1.0f, -1.0f, 0.0f, // bottom right
1.0f, 1.0f, 0.0f // top right
};
static float mUVCoords[] = {
0, 0, // top left
0, 1, // bottom left
1, 1, // bottom right
1, 0 // top right
};
private short mVertexIndex[] = {0, 1, 2, 0, 2, 3}; // order to draw vertices
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;"
+ "attribute vec4 aPosition;\n"
+ "attribute vec2 aTextureCoord;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main() {\n"
+ " gl_Position = uMVPMatrix * aPosition;\n"
+ " vTextureCoord = aTextureCoord;\n"
+ "}\n";
private final String fragmentShaderCode =
"precision mediump float;\n"
+ "uniform sampler2D Ytex;\n"
+ "uniform sampler2D Utex,Vtex;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main(void) {\n"
+ " float nx,ny,r,g,b,y,u,v;\n"
+ " mediump vec4 txl,ux,vx;"
+ " nx=vTextureCoord[0];\n"
+ " ny=vTextureCoord[1];\n"
+ " y=texture2D(Ytex,vec2(nx,ny)).r;\n"
+ " u=texture2D(Utex,vec2(nx,ny)).r;\n"
+ " v=texture2D(Vtex,vec2(nx,ny)).r;\n"
+ " y=1.1643*(y-0.0625);\n"
+ " u=u-0.5;\n"
+ " v=v-0.5;\n"
+ " r=y+1.5958*v;\n"
+ " g=y-0.39173*u-0.81290*v;\n"
+ " b=y+2.017*u;\n"
// --> Bilateral blur filter code HERE <--
+ " gl_FragColor=vec4(r,g,b,1.0);\n"
+ "}\n";
ReentrantLock mFrameLock = new ReentrantLock();
Frame mCurrentFrame;
private int mProgram;
private int mTextureWidth;
private int mTextureHeight;
private int mViewportWidth;
private int mViewportHeight;
public MyRenderer()
{
ByteBuffer bb = ByteBuffer.allocateDirect(mXYZCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mXYZCoords);
mVertexBuffer.position(0);
ByteBuffer tb = ByteBuffer.allocateDirect(mUVCoords.length * 4);
tb.order(ByteOrder.nativeOrder());
mTextureBuffer = tb.asFloatBuffer();
mTextureBuffer.put(mUVCoords);
mTextureBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(mVertexIndex.length * 2);
dlb.order(ByteOrder.nativeOrder());
mDrawListBuffer = dlb.asShortBuffer();
mDrawListBuffer.put(mVertexIndex);
mDrawListBuffer.position(0);
}
@Override public void onSurfaceCreated(GL10 gl, EGLConfig config)
{
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram);
int positionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
int textureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4, mVertexBuffer);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(textureHandle, TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false, TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer);
GLES20.glEnableVertexAttribArray(textureHandle);
GLES20.glUseProgram(mProgram);
int i = GLES20.glGetUniformLocation(mProgram, "Ytex"); // GLES20.glUniform3i(i, 0, 1, 2);
GLES20.glUniform1i(i, 0); /* Bind Ytex to texture unit 0 */
i = GLES20.glGetUniformLocation(mProgram, "Utex");
GLES20.glUniform1i(i, 1); /* Bind Utex to texture unit 1 */
i = GLES20.glGetUniformLocation(mProgram, "Vtex");
GLES20.glUniform1i(i, 2); /* Bind Vtex to texture unit 2 */
mTextureWidth = 0;
mTextureHeight = 0;
}
static void initializeTexture(int name, int id, int width, int height)
{
GLES20.glActiveTexture(name);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width, height, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
}
void setupTextures(Frame frame)
{
if (mTextureIds[0] != 0)
{
GLES20.glDeleteTextures(3, mTextureIds, 0);
}
GLES20.glGenTextures(3, mTextureIds, 0);
int w = frame.getWidth();
int h = frame.getHeight();
int hw = (w + 1) >> 1;
int hh = (h + 1) >> 1;
initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h);
initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh);
initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh);
mTextureWidth = frame.getWidth();
mTextureHeight = frame.getHeight();
}
void updateTextures(Frame frame)
{
int width = frame.getWidth();
int height = frame.getHeight();
int half_width = (width + 1) >> 1;
int half_height = (height + 1) >> 1;
int y_size = width * height;
int uv_size = half_width * half_height;
ByteBuffer bb = frame.getBuffer();
bb.clear(); // If we are reusing this frame, make sure we reset position and limit
if (bb.remaining() == y_size + uv_size * 2)
{
bb.position(0);
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size + uv_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
int i = GLES20.glGetUniformLocation(mProgram, "width");
GLES20.glUniform1f(i, (float) mTextureWidth);
i = GLES20.glGetUniformLocation(mProgram, "height");
GLES20.glUniform1f(i, (float) mTextureHeight);
}
else
{
mTextureWidth = 0;
mTextureHeight = 0;
}
}
@Override public void onSurfaceChanged(GL10 gl, int width, int height)
{
GLES20.glViewport(0, 0, width, height);
mViewportWidth = width;
mViewportHeight = height;
}
@Override public void onDrawFrame(GL10 gl)
{
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
mFrameLock.lock();
if (mCurrentFrame != null && !mVideoDisabled)
{
GLES20.glUseProgram(mProgram);
if (mTextureWidth != mCurrentFrame.getWidth() || mTextureHeight != mCurrentFrame.getHeight())
{
setupTextures(mCurrentFrame);
}
updateTextures(mCurrentFrame);
Matrix.setIdentityM(mScaleMatrix, 0);
float scaleX = 1.0f, scaleY = 1.0f;
float ratio = (float) mCurrentFrame.getWidth() / mCurrentFrame.getHeight();
float vratio = (float) mViewportWidth / mViewportHeight;
if (mVideoFitEnabled)
{
if (ratio > vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
else
{
if (ratio < vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
Matrix.scaleM(mScaleMatrix, 0, scaleX * (mCurrentFrame.isMirroredX() ? -1.0f : 1.0f), scaleY, 1);
int mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mScaleMatrix, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
}
mFrameLock.unlock();
}
public void displayFrame(Frame frame)
{
mFrameLock.lock();
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = frame;
mFrameLock.unlock();
}
public static int loadShader(int type, String shaderCode)
{
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
public void disableVideo(boolean b)
{
mFrameLock.lock();
mVideoDisabled = b;
if (mVideoDisabled)
{
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = null;
}
mFrameLock.unlock();
}
public void enableVideoFit(boolean enableVideoFit)
{
mVideoFitEnabled = enableVideoFit;
}
}
您要做的是 downsampling.What 您需要做的是首先渲染成一个小的 FBO，其中视口和 FBO 附件的大小相同。在该通道中，您可以应用模糊效果。然后您可以将其渲染或 blit 到另一个原始大小的 FBO 以获得缩放的纹理 back.It 重要的是要注意，根据您的蓝色技术，放大后的结果可能会有明显的质量下降。
由于某些 OpenGL API 调用在 Android SDK 中可能有不同的命名，这里是您需要执行的一般伪代码：
设 w = 原始宽度，h = 原始高度。
1.Create 带有大小为 w/2 和 h/2 的纹理附件的自定义 FBO（如果您计划以原始分辨率的一半进行下采样。
2.Attach纹理到FBO.BindFBO写入。
3.set glViewport() 的大小与 FBO 纹理附件相同。
4.Render 全屏 quad.Apply 你在本关中的效果。
5.Second pass:Bind 返回默认帧缓冲区（或者 - 如果您稍后有其他渲染阶段，则使用另一个自定义 FBO）。如果您只想 blit，请确保绑定第一个 FBO只读不写。
6.Bind 从上一次传递到采样器单元附加到 FBO 的纹理。
7.Set w 和 h 的视口 size.Draw 全屏 quad.Do 在片段着色器中无论你需要什么。（伽玛校正，混合等..）
it.What 在这种技术中的好处在于 OpenGL 会自动为您执行 upscale/downscale 过滤类型，您 select 用于处理过程中涉及的纹理。
另一方面，note:If 你有性能问题，在 Java.Though 中编写渲染器不是一个好主意，你的问题可能是由于错误的 API 用法或模糊效果的低效算法应用。
最终我弄明白了，感谢一位非常有才华的人帮助了我。 这是我的渲染器 class，它现在有一个带有 2 个渲染通道的 frameBuffer：
static class MyRenderer implements GLSurfaceView.Renderer
{
int mTextureIds[] = new int[4];
float[] mScaleMatrix = new float[16];
float[] mFilterScaleMatrix = new float[16];
private FloatBuffer mVertexBuffer;
private FloatBuffer mTextureBuffer;
private ShortBuffer mDrawListBuffer;
private IntBuffer frameBuffer;
boolean mVideoFitEnabled = true;
boolean mVideoDisabled = false;
// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static final int TEXTURECOORDS_PER_VERTEX = 2;
static float mXYZCoords[] = {
-1.0f, 1.0f, 0.0f, // top left
-1.0f, -1.0f, 0.0f, // bottom left
1.0f, -1.0f, 0.0f, // bottom right
1.0f, 1.0f, 0.0f // top right
};
static float mUVCoords[] = {
0, 0, // top left
0, 1, // bottom left
1, 1, // bottom right
1, 0 // top right
};
private short mVertexIndex[] = {0, 1, 2, 0, 2, 3}; // order to draw vertices
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;"
+ "attribute vec4 aPosition;\n"
+ "attribute vec2 aTextureCoord;\n"
+ "varying vec2 vTextureCoord;\n"
+ "void main() {\n"
+ " gl_Position = uMVPMatrix * aPosition;\n"
+ " vTextureCoord = aTextureCoord;\n"
+ "}\n";
private final String fragmentShaderCode =
"YUV to RGB Conversion shader HERE";
private final String frameBufferShader =
"MY filter effect shader HERE";
ReentrantLock mFrameLock = new ReentrantLock();
Frame mCurrentFrame;
private int mProgram;
private int mProgramFilter;
private int mTextureWidth;
private int mTextureHeight;
private int mViewportWidth;
private int mViewportHeight;
public MyRenderer()
{
ByteBuffer bb = ByteBuffer.allocateDirect(mXYZCoords.length * 4);
bb.order(ByteOrder.nativeOrder());
mVertexBuffer = bb.asFloatBuffer();
mVertexBuffer.put(mXYZCoords);
mVertexBuffer.position(0);
ByteBuffer tb = ByteBuffer.allocateDirect(mUVCoords.length * 4);
tb.order(ByteOrder.nativeOrder());
mTextureBuffer = tb.asFloatBuffer();
mTextureBuffer.put(mUVCoords);
mTextureBuffer.position(0);
ByteBuffer dlb = ByteBuffer.allocateDirect(mVertexIndex.length * 2);
dlb.order(ByteOrder.nativeOrder());
mDrawListBuffer = dlb.asShortBuffer();
mDrawListBuffer.put(mVertexIndex);
mDrawListBuffer.position(0);
frameBuffer = IntBuffer.allocate(1);
}
@Override public void onSurfaceCreated(GL10 gl, EGLConfig config)
{
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
int filterVertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int filterFragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, frameBufferShader);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram);
mProgramFilter = GLES20.glCreateProgram(); // create empty OpenGL ES Program
GLES20.glAttachShader(mProgramFilter, filterVertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgramFilter, filterFragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgramFilter);
int positionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
int textureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX, GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4, mVertexBuffer);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(textureHandle, TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false, TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer);
GLES20.glEnableVertexAttribArray(textureHandle);
GLES20.glUseProgram(mProgram);
int i = GLES20.glGetUniformLocation(mProgram, "Ytex");
GLES20.glUniform1i(i, 3); /* Bind Ytex to texture unit 0 */
i = GLES20.glGetUniformLocation(mProgram, "Utex");
GLES20.glUniform1i(i, 1); /* Bind Utex to texture unit 1 */
i = GLES20.glGetUniformLocation(mProgram, "Vtex");
GLES20.glUniform1i(i, 2); /* Bind Vtex to texture unit 2 */
GLES20.glUseProgram(mProgramFilter);
i = GLES20.glGetUniformLocation(mProgramFilter, "Ytex");
GLES20.glUniform1i(i, 0);
mTextureWidth = 0;
mTextureHeight = 0;
}
static void initializeTexture(int name, int id, int width, int height)
{
GLES20.glActiveTexture(name);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width, height, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null);
}
void setupTextures(Frame frame)
{
if (mTextureIds[0] != 0)
{
GLES20.glDeleteTextures(4, mTextureIds, 0);
}
GLES20.glGenTextures(4, mTextureIds, 0);
int w = frame.getWidth();
int h = frame.getHeight();
int hw = (w + 1) >> 1;
int hh = (h + 1) >> 1;
initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h);
initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh);
initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh);
GLES20.glGenFramebuffers(1, frameBuffer);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer.get(0));
GLES20.glActiveTexture(GLES20.GL_TEXTURE3);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[3]);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, w, h, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, mTextureIds[3], 0);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
mTextureWidth = frame.getWidth();
mTextureHeight = frame.getHeight();
GLES20.glUseProgram(mProgramFilter);
int i = GLES20.glGetUniformLocation(mProgramFilter, "width");
GLES20.glUniform1f(i, (float) mTextureWidth);
i = GLES20.glGetUniformLocation(mProgramFilter, "height");
GLES20.glUniform1f(i, (float) mTextureHeight);
}
void updateTextures(Frame frame)
{
int width = frame.getWidth();
int height = frame.getHeight();
int half_width = (width + 1) >> 1;
int half_height = (height + 1) >> 1;
int y_size = width * height;
int uv_size = half_width * half_height;
ByteBuffer bb = frame.getBuffer();
bb.clear(); // If we are reusing this frame, make sure we reset position and limit
if (bb.remaining() == y_size + uv_size * 2)
{
bb.position(0);
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width, height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
bb.position(y_size + uv_size);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2]);
GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, half_width, half_height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, bb);
}
else
{
mTextureWidth = 0;
mTextureHeight = 0;
}
}
@Override public void onSurfaceChanged(GL10 gl, int width, int height)
{
/// GLES20.glViewport(0, 0, width, height);
mViewportWidth = width;
mViewportHeight = height;
}
@Override public void onDrawFrame(GL10 gl)
{
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
mFrameLock.lock();
if (mCurrentFrame != null && !mVideoDisabled)
{
if (mTextureWidth != mCurrentFrame.getWidth() || mTextureHeight != mCurrentFrame.getHeight())
{
setupTextures(mCurrentFrame);
}
updateTextures(mCurrentFrame);
/// Step 1: Smoothing Filter - Render to FrameBuffer [pass 1]
Matrix.setIdentityM(mFilterScaleMatrix, 0);
GLES20.glViewport(0, 0, mTextureWidth, mTextureHeight);
GLES20.glUseProgram(mProgramFilter);
int mMVPFilterMatrixHandle = GLES20.glGetUniformLocation(mProgramFilter, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPFilterMatrixHandle, 1, false, mFilterScaleMatrix, 0);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer.get(0));
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
/// Step 2: Draw + RGB Conversion - Render to screen [pass 2]
Matrix.setIdentityM(mScaleMatrix, 0);
float scaleX = 1.0f, scaleY = 1.0f;
float ratio = (float) mCurrentFrame.getWidth() / mCurrentFrame.getHeight();
float vratio = (float) mViewportWidth / mViewportHeight;
if (mVideoFitEnabled)
{
if (ratio > vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
else
{
if (ratio < vratio)
{
scaleY = vratio / ratio;
}
else
{
scaleX = ratio / vratio;
}
}
Matrix.scaleM(mScaleMatrix, 0, scaleX * (mCurrentFrame.isMirroredX() ? -1.0f : 1.0f), scaleY, 1);
GLES20.glUseProgram(mProgram);
GLES20.glViewport(0, 0, mViewportWidth, mViewportHeight);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
int mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false, mScaleMatrix, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);
}
mFrameLock.unlock();
}
public void displayFrame(Frame frame)
{
mFrameLock.lock();
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = frame;
mFrameLock.unlock();
}
public static int loadShader(int type, String shaderCode)
{
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
public void disableVideo(boolean b)
{
mFrameLock.lock();
mVideoDisabled = b;
if (mVideoDisabled)
{
if (this.mCurrentFrame != null)
{
this.mCurrentFrame.recycle();
}
this.mCurrentFrame = null;
}
mFrameLock.unlock();
}
public void enableVideoFit(boolean enableVideoFit)
{
mVideoFitEnabled = enableVideoFit;
}
}