nvidia wayland: support hardware cursor plane

include/color_conversion.h

@@ -37,6 +37,6 @@ typedef struct {
 int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conversion_params *params);
 void gsr_color_conversion_deinit(gsr_color_conversion *self);
 
-int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation);
+int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture);
 
 #endif /* GSR_COLOR_CONVERSION_H */

src/capture/kms_cuda.c

@@ -17,7 +17,7 @@
     and copy the input textures to the pixel buffer objects. Use sw_format NV12 as well. Then this is
     similar to kms_vaapi. This allows us to remove one extra texture and texture copy.
 */
-/* TODO: Support cursor plane capture when nvidia supports cursor plane */
+// TODO: Wayland capture
 
 #define MAX_CONNECTOR_IDS 32
 
@@ -48,7 +48,9 @@ typedef struct {
     CUarray mapped_array;
 
     unsigned int input_texture;
+    unsigned int cursor_texture;
     unsigned int target_texture;
+
     gsr_color_conversion color_conversion;
 } gsr_capture_kms_cuda;
 
@@ -275,6 +277,14 @@ static void gsr_capture_kms_cuda_tick(gsr_capture *cap, AVCodecContext *video_co
         cap_kms->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
         cap_kms->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
+        cap_kms->params.egl->glGenTextures(1, &cap_kms->cursor_texture);
+        cap_kms->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, cap_kms->cursor_texture);
+        cap_kms->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+        cap_kms->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+        cap_kms->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+        cap_kms->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+        cap_kms->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+
         cap_kms->target_texture = gl_create_texture(cap_kms, video_codec_context->width, video_codec_context->height);
         if(cap_kms->target_texture == 0) {
             fprintf(stderr, "gsr error: gsr_capture_kms_cuda_tick: failed to create opengl texture\n");
@@ -486,7 +496,33 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
     gsr_color_conversion_draw(&cap_kms->color_conversion, cap_kms->input_texture,
         (vec2i){0, 0}, cap_kms->capture_size,
         capture_pos, cap_kms->capture_size,
-        0.0f);
+        0.0f, false);
+
+    if(cursor_drm_fd) {
+        const intptr_t img_attr_cursor[] = {
+            EGL_LINUX_DRM_FOURCC_EXT,       cursor_drm_fd->pixel_format,
+            EGL_WIDTH,                      cursor_drm_fd->width,
+            EGL_HEIGHT,                     cursor_drm_fd->height,
+            EGL_DMA_BUF_PLANE0_FD_EXT,      cursor_drm_fd->fd,
+            EGL_DMA_BUF_PLANE0_OFFSET_EXT,  cursor_drm_fd->offset,
+            EGL_DMA_BUF_PLANE0_PITCH_EXT,   cursor_drm_fd->pitch,
+            EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, cursor_drm_fd->modifier & 0xFFFFFFFFULL,
+            EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, cursor_drm_fd->modifier >> 32ULL,
+            EGL_NONE
+        };
+
+        EGLImage cursor_image = cap_kms->params.egl->eglCreateImage(cap_kms->params.egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr_cursor);
+        cap_kms->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, cap_kms->cursor_texture);
+        cap_kms->params.egl->glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, cursor_image);
+        cap_kms->params.egl->eglDestroyImage(cap_kms->params.egl->egl_display, cursor_image);
+        cap_kms->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+
+        vec2i cursor_size = {cursor_drm_fd->width, cursor_drm_fd->height};
+        gsr_color_conversion_draw(&cap_kms->color_conversion, cap_kms->cursor_texture,
+            (vec2i){cursor_drm_fd->x, cursor_drm_fd->y}, cursor_size,
+            (vec2i){0, 0}, cursor_size,
+            0.0f, true);
+    }
 
     cap_kms->params.egl->eglSwapBuffers(cap_kms->params.egl->egl_display, cap_kms->params.egl->egl_surface);
 
@@ -539,6 +575,11 @@ static void gsr_capture_kms_cuda_stop(gsr_capture *cap, AVCodecContext *video_co
             cap_kms->input_texture = 0;
         }
 
+        if(cap_kms->cursor_texture) {
+            cap_kms->params.egl->glDeleteTextures(1, &cap_kms->cursor_texture);
+            cap_kms->cursor_texture = 0;
+        }
+
         if(cap_kms->target_texture) {
             cap_kms->params.egl->glDeleteTextures(1, &cap_kms->target_texture);
             cap_kms->target_texture = 0;

src/capture/kms_vaapi.c

@@ -505,7 +505,7 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
         gsr_color_conversion_draw(&cap_kms->color_conversion, cap_kms->input_texture,
             (vec2i){0, 0}, cap_kms->capture_size,
             (vec2i){0, 0}, cap_kms->capture_size,
-            0.0f);
+            0.0f, false);
     } else {
         if(!capture_is_combined_plane)
             capture_pos = (vec2i){drm_fd->x, drm_fd->y};
@@ -515,7 +515,7 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
         gsr_color_conversion_draw(&cap_kms->color_conversion, cap_kms->input_texture,
             (vec2i){0, 0}, cap_kms->capture_size,
             capture_pos, cap_kms->capture_size,
-            texture_rotation);
+            texture_rotation, false);
 
         if(cursor_drm_fd) {
             const intptr_t img_attr_cursor[] = {
@@ -540,7 +540,7 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
             gsr_color_conversion_draw(&cap_kms->color_conversion, cap_kms->cursor_texture,
                 (vec2i){cursor_drm_fd->x, cursor_drm_fd->y}, cursor_size,
                 (vec2i){0, 0}, cursor_size,
-                0.0f);
+                0.0f, false);
         }
     }
 

src/capture/xcomposite_vaapi.c

@@ -442,7 +442,7 @@ static int gsr_capture_xcomposite_vaapi_capture(gsr_capture *cap, AVFrame *frame
     gsr_color_conversion_draw(&cap_xcomp->color_conversion, window_texture_get_opengl_texture_id(&cap_xcomp->window_texture),
         (vec2i){0, 0}, cap_xcomp->texture_size,
         (vec2i){0, 0}, cap_xcomp->texture_size,
-        texture_rotation);
+        texture_rotation, false);
 
     cap_xcomp->params.egl->eglSwapBuffers(cap_xcomp->params.egl->egl_display, cap_xcomp->params.egl->egl_surface);
     //cap_xcomp->params.egl->glFlush();

src/color_conversion.c

@@ -4,6 +4,8 @@
 #include <math.h>
 #include <assert.h>
 
+/* TODO: highp instead of mediump? */
+
 #define MAX_SHADERS 2
 #define MAX_FRAMEBUFFERS 2
 
@@ -59,6 +61,43 @@ static int load_shader_bgr(gsr_shader *shader, gsr_egl *egl, int *rotation_unifo
     return 0;
 }
 
+static int load_shader_bgr_external_texture(gsr_shader *shader, gsr_egl *egl, int *rotation_uniform) {
+    char vertex_shader[2048];
+    snprintf(vertex_shader, sizeof(vertex_shader),
+        "#version 300 es                                   \n"
+        "in vec2 pos;                                      \n"
+        "in vec2 texcoords;                                \n"
+        "out vec2 texcoords_out;                           \n"
+        "uniform float rotation;                           \n"
+        ROTATE_Z
+        "void main()                                       \n"
+        "{                                                 \n"
+        "  texcoords_out = texcoords;                      \n"
+        "  gl_Position = vec4(pos.x, pos.y, 0.0, 1.0) * rotate_z(rotation);    \n"
+        "}                                                 \n");
+
+    char fragment_shader[] =
+        "#version 300 es                                                                 \n"
+        "#extension GL_OES_EGL_image_external : enable                                   \n"
+        "#extension GL_OES_EGL_image_external_essl3 : require                            \n"
+        "precision mediump float;                                                        \n"
+        "in vec2 texcoords_out;                                                          \n"
+        "uniform samplerExternalOES tex1;                                                \n"
+        "out vec4 FragColor;                                                             \n"
+        "void main()                                                                     \n"
+        "{                                                                               \n"
+        "  FragColor = texture(tex1, texcoords_out).bgra;                                \n"
+        "}                                                                               \n";
+
+    if(gsr_shader_init(shader, egl, vertex_shader, fragment_shader) != 0)
+        return -1;
+
+    gsr_shader_bind_attribute_location(shader, "pos", 0);
+    gsr_shader_bind_attribute_location(shader, "texcoords", 1);
+    *rotation_uniform = egl->glGetUniformLocation(shader->program_id, "rotation");
+    return 0;
+}
+
 static int load_shader_y(gsr_shader *shader, gsr_egl *egl, int *rotation_uniform) {
     char vertex_shader[2048];
     snprintf(vertex_shader, sizeof(vertex_shader),
@@ -202,6 +241,11 @@ int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conver
                 fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load bgr shader\n");
                 goto err;
             }
+
+            if(load_shader_bgr_external_texture(&self->shaders[1], self->params.egl, &self->rotation_uniforms[1]) != 0) {
+                fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load bgr shader (external texture)\n");
+                goto err;
+            }
             break;
         }
         case GSR_DESTINATION_COLOR_NV12: {
@@ -263,18 +307,25 @@ void gsr_color_conversion_deinit(gsr_color_conversion *self) {
 }
 
 /* |source_pos| is in pixel coordinates and |source_size|  */
-int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation) {
+int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture) {
     /* TODO: Do not call this every frame? */
     vec2i dest_texture_size = {0, 0};
     self->params.egl->glBindTexture(GL_TEXTURE_2D, self->params.destination_textures[0]);
     self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &dest_texture_size.x);
     self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &dest_texture_size.y);
+    self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
+
+    const int texture_target = external_texture ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
 
-    /* TODO: Do not call this every frame? */
     vec2i source_texture_size = {0, 0};
-    self->params.egl->glBindTexture(GL_TEXTURE_2D, texture_id);
+    if(external_texture) {
-    self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &source_texture_size.x);
+        source_texture_size = source_size;
-    self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &source_texture_size.y);
+    } else {
+        /* TODO: Do not call this every frame? */
+        self->params.egl->glBindTexture(texture_target, texture_id);
+        self->params.egl->glGetTexLevelParameteriv(texture_target, 0, GL_TEXTURE_WIDTH, &source_texture_size.x);
+        self->params.egl->glGetTexLevelParameteriv(texture_target, 0, GL_TEXTURE_HEIGHT, &source_texture_size.y);
+    }
 
     if(abs_f(M_PI * 0.5f - rotation) <= 0.001f || abs_f(M_PI * 1.5f - rotation) <= 0.001f) {
         float tmp = source_texture_size.x;
@@ -314,7 +365,7 @@ int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_i
 
     self->params.egl->glBindVertexArray(self->vertex_array_object_id);
     self->params.egl->glViewport(0, 0, dest_texture_size.x, dest_texture_size.y);
-    self->params.egl->glBindTexture(GL_TEXTURE_2D, texture_id);
+    self->params.egl->glBindTexture(texture_target, texture_id);
 
     /* TODO: this, also cleanup */
     //self->params.egl->glBindBuffer(GL_ARRAY_BUFFER, self->vertex_buffer_object_id);
@@ -324,8 +375,13 @@ int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_i
         self->params.egl->glBindFramebuffer(GL_FRAMEBUFFER, self->framebuffers[0]);
         //cap_xcomp->params.egl->glClear(GL_COLOR_BUFFER_BIT); // TODO: Do this in a separate clear_ function. We want to do that when using multiple drm to create the final image (multiple monitors for example)
 
-        gsr_shader_use(&self->shaders[0]);
+        if(external_texture) {
-        self->params.egl->glUniform1f(self->rotation_uniforms[0], rotation);
+            gsr_shader_use(&self->shaders[1]);
+            self->params.egl->glUniform1f(self->rotation_uniforms[1], rotation);
+        } else {
+            gsr_shader_use(&self->shaders[0]);
+            self->params.egl->glUniform1f(self->rotation_uniforms[0], rotation);
+        }
         self->params.egl->glDrawArrays(GL_TRIANGLES, 0, 6);
     }
 
@@ -340,7 +396,7 @@ int gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_i
 
     self->params.egl->glBindVertexArray(0);
     gsr_shader_use_none(&self->shaders[0]);
-    self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
+    self->params.egl->glBindTexture(texture_target, 0);
     self->params.egl->glBindFramebuffer(GL_FRAMEBUFFER, 0);
     return 0;
 }

src/window_texture.c

@@ -120,4 +120,4 @@ int window_texture_on_resize(WindowTexture *self) {
 
 unsigned int window_texture_get_opengl_texture_id(WindowTexture *self) {
     return self->texture_id;
-}
+}