我目前正在使用 ffmpeg 连接和解码视频流的 android 上的 rtsp 播放器。我想使用OpenGL es 2.0 将YUV帧转换为RGB帧并显示它但是我被阻止了(这是我第一次使用opengl)。

我会尽量解释清楚我的问题是什么。

从 NDK android 我使用这个方法初始化一个 opengl 上下文(从我想用来显示图像的线程):

    //
EGLint attribs[] = {
        EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
        EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
        EGL_BLUE_SIZE, 8,
        EGL_GREEN_SIZE, 8,
        EGL_RED_SIZE, 8,
        EGL_ALPHA_SIZE, 8,
        EGL_NONE
};
EGLint contextAttrs[] = {
        EGL_CONTEXT_CLIENT_VERSION, 2,
        EGL_NONE
};

LOGI("Initializing context");

if((display = eglGetDisplay(EGL_DEFAULT_DISPLAY)) == EGL_NO_DISPLAY)
{
    closeContext();
    return;
}

if(!eglInitialize(display, 0, 0))
{
    closeContext();
    return;
}

if(!eglChooseConfig(display, attribs, &config, 1, &numConfigs))
{
    closeContext();
    return;
}

if(!eglGetConfigAttrib(display, config, EGL_NATIVE_VISUAL_ID, &format))
{
    closeContext();
    return;
}

ANativeWindow_setBuffersGeometry(window, 0, 0, format);

if(!(surface = eglCreateWindowSurface(display, config, window, 0)))
{
    closeContext();
    return;
}

if(!(context = eglCreateContext(display, config, 0, contextAttrs)))
{
    closeContext();
    return;
}

if(!eglMakeCurrent(display, surface, surface, context))
{
    closeContext();
    return;
}

if(!eglQuerySurface(display, surface, EGL_WIDTH, &width) || !eglQuerySurface(display, surface, EGL_HEIGHT, &height))
{
    closeContext();
    return;
}

LOGI("EGLWIDTH : %d EGLHEIGHT : %d ", (int)width, (int)height);

isInitEGLContext = 1;

然后我使用这种方法设置图形:

    //
//Load Vertex and Fragment Shader, attach shader and link program
 programId = createProgram(kVertexShader, kFragmentShader);
 LOGI("Program id : %d error :  %d",(int) programId, glGetError());


if(!programId)
{
    LOGI("Could not create program");
    return;
}
// get index of the generic vertex attribute bound to vPosition
positionObject = (int) glGetAttribLocation(programId, "vPosition");

 // get index of the generic vertex attribute bound to vTexCoord
texturePosition = (int) glGetAttribLocation(programId, "vTexCoord");

// get the location of yTexture within the program (corresponding to   program id)
yuv_texture_object[0] = glGetUniformLocation(programId, "yTexture");

// get the location of uTexture within the program
yuv_texture_object[1] = glGetUniformLocation(programId, "uTexture");

// get the location of vTexture within the program
yuv_texture_object[2] = glGetUniformLocation(programId, "vTexture");

// Setup width of each planes (display size)
stream_yuv_width[0] = 800;
stream_yuv_width[1] = 400;
stream_yuv_width[2] = 400;

// Setup height of each planes (display size)
stream_yuv_height[0] = 600;
stream_yuv_height[1] = 300;
stream_yuv_height[2] = 300;

//set the view port
glViewport(0,0,stream_yuv_width[0],stream_yuv_height[0]);
LOGI("glViewPort() %d ", glGetError());

我已经硬编码了显示尺寸(现在)，直到我得到一些有用的东西。

createProgram 方法，加载着色器，创建程序，编译和链接着色器成功。

这是我的着色器:

const char kVertexShader[] =
"attribute vec4 vPosition;\n"
"attribute vec2 vTexCoord;\n"
"varying vec2 v_vTexCoord;\n"
"void main() {\n"
"gl_Position = vPosition;\n"
"v_vTexCoord = vTexCoord;\n"
"}\n";

const char kFragmentShader[] =
"precision mediump float; \n"
"varying vec2 v_vTexCoord;\n"
"uniform sampler2D yTexture;\n"
"uniform sampler2D uTexture;\n"
"uniform sampler2D vTexture;\n"
"void main() {\n"
"float nx, ny; \n"
"nx = v_vTexCoord.x; \n"
"ny = v_vTexCoord.y; \n"
"float y=texture2D(yTexture, v_vTexCoord).r;\n"
"float u=texture2D(uTexture, vec2(nx / 2.0, ny / 2.0)).r;\n"
"float v=texture2D(vTexture, vec2(nx / 2.0, ny / 2.0)).r;\n"
"y = 1.1643 * (y - 0.0625);\n"
"u = u - 0.5; \n"
"v = v - 0.5; \n"
"float r=y + 1.5958 * v;\n"
"float g=y - 0.39173 * u - 0.81290 * v;\n"
"float b=y + 2.017 * u;\n"
"gl_FragColor = vec4(r, g, b, 1.0);\n"
"}\n";

const GLfloat kVertexInformation[] = {
    -1.0f, 1.0f,           // TexCoord 0 top left
    -1.0f,-1.0f,           // TexCoord 1 bottom left
    1.0f,-1.0f,           // TexCoord 2 bottom right
    1.0f, 1.0f            // TexCoord 3 top right
};
const GLshort kTextureCoordinateInformation[] = {
    0, 0,         // TexCoord 0 top left
    0, 1,         // TexCoord 1 bottom left
    1, 1,         // TexCoord 2 bottom right
    1, 0          // TexCoord 3 top right  
};
const GLuint kStride = 0;//COORDS_PER_VERTEX * 4;
const GLshort kIndicesInformation[] = {
    0, 1, 2,
    0, 2, 3
};

然后我设置 yuv 纹理并渲染到纹理，此时 yuv_width[i] 和 yuv_height[i] 设置为正确的值:

void setupYUVTexture()
{
//Setup the pixel alignement
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
LOGI("glPixelStorei() : %d ", glGetError());
int i = 0;

for(i = 0 ; i < 3 ; ++i)
{
    //Check if the texture already setup
    if(yuv_texture_id[i] != 0)
    {
        glDeleteTextures(1, &yuv_texture_id[i]);
        yuv_texture_id[i] = 0;
    }
    // Active the i texture
    glActiveTexture(GL_TEXTURE0 + i);

    //Generate the texture name
    glGenTextures(1, &yuv_texture_id[i]);

    // Bind the texture
    glBindTexture(GL_TEXTURE_2D, yuv_texture_id[i]);

   // Setup the texture parameters
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);  
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);   
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);


    //Define the texture image
    glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuv_width[i], yuv_height[i], 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
    LOGI("glTexImage2D() %d ", glGetError());
}
}

void renderToTexture()
{
 // Generate framebuffer object name
    glGenFramebuffers(1, &frameBufferObject);
    //Bind the framebuffer
    glBindFramebuffer(GL_FRAMEBUFFER, frameBufferObject);


    //Generate render buffer object name
    glGenRenderbuffers(1, &renderBufferObject);

    //Bind render buffer
    glBindRenderbuffer(GL_RENDERBUFFER, renderBufferObject);

   //Create and initialize render buffer for display RGBA with the same size of the viewport
    glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA4, 800, 600);

    //Attach render buffer to frame buffer object
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, renderBufferObject);

    //Attach y plane to frame buffer object
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[0], 0);

   //Attach u plane to frame buffer object
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[1], 0);

     //Attach v plane to frame buffer object
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, yuv_texture_id[2], 0);

    // Bind the framebuffer
    glBindFramebuffer(GL_FRAMEBUFFER, 0);

    //Check if the framebuffer is correctly setup
    GLint status = glCheckFramebufferStatus(GL_FRAMEBUFFER);

    if(status != GL_FRAMEBUFFER_COMPLETE)
    {
        LOGI(" FBO setting fault : %d ", status);
        return;
    }
}

最后，我的画框方法:

void drawFrame()
{
LOGI("DrawFrame");
glBindFramebuffer(GL_FRAMEBUFFER, frameBufferObject);
printGLError("glBindFramebuffer");
glUseProgram(programId);
printGLError("glUseProgram");
int i = 0;
for(i = 0 ; i < 3 ; ++i)
{
    glActiveTexture(GL_TEXTURE0 + i);
    printGLError("glActiveTexture");

    glBindTexture(GL_TEXTURE_2D, yuv_texture_object[i]);
    printGLError("glBindTexture");

    glUniform1i(yuv_texture_object[i], i);
    printGLError("glUniform1i");            
    LOGI("Plan : %d Largeur : %d Hauteur : %d ", i, yuv_width[i], yuv_height[i]);
    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0,yuv_width[i], yuv_height[i], GL_LUMINANCE, GL_UNSIGNED_BYTE, yuv_planes[i]);
            printGLError("glTexSubImage2D");

    }

    glVertexAttribPointer(positionObject, 2, GL_FLOAT, GL_FALSE, kStride, kVertexInformation);
    printGLError("glVertexAttribPointer");

    glVertexAttribPointer(texturePosition, 2, GL_SHORT, GL_FALSE, kStride, kTextureCoordinateInformation);
    printGLError("glVertexAttribPointer");

    glEnableVertexAttribArray(positionObject);      
    printGLError("glVertexAttribArray");

    glEnableVertexAttribArray(texturePosition);
    printGLError("glVertexAttribArray");

    glBindFramebuffer(GL_FRAMEBUFFER, 0);
    printGLError("glBindFramebuffer");

    glDrawElements(GL_TRIANGLE_STRIP, 6, GL_UNSIGNED_SHORT, kIndicesInformation);
    printGLError("glDrawElements");

    eglSwapBuffers(display, surface);
    printGLError("eglSwapBuffers");

}

我初始化一次opengl纹理和其他必要的属性，然后当一个帧被解码时，我将y缓冲区重新复制到yuv_planes[0]，u缓冲区在yuv_planes[1] 和 yuv_planes[2] 中的 v 缓冲区。

一旦使用 ffmpeg 正确解码一帧，我将按以下顺序调用:
- initContext()
- setupGraphics()
- setupYUVTexture()
- renderToTexture()

然后我调用 drawFrame。当然，当一切都初始化时，我会在每个解码帧之后直接调用 drawFrame。

这是我现在的输出。

图像的大小是正确的，但现在我在这里被阻塞了。我不明白为什么图像显示是绿色的! 任何想法

最佳答案

这需要处理很多代码，还有很多可能出错的事情 ;)。要调试这类问题，我会一步一步来。

1. 只需输出红色 (gl_FragColor = vec4(1.0, 0.5, 0.5, 1.0)) 以确保您的配置正常工作。
2. 尝试以灰度输出每个纹理。 (gl_FragColor = vec4(y, y, y, 1.0))
3. 如果一切正常，则很可能意味着您的 yuv => rgb 转换在某处有误。
4. 如果这不起作用，那么我会怀疑纹理映射中存在某些问题。仔细检查您的 glTexSubImage2D 调用。您可能需要使用不同的步幅或使用不同的坐标系。

关于android - YUV 到 RGB 转换和显示使用 opengl es 2.0 从 android ndk 使用着色器，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/34764238/