Properly cut out cursor outside video area when dealing with hevc amd padding

include/capture/xcomposite.h

@@ -37,7 +37,6 @@ typedef struct {
     Atom net_active_window_atom;
 
     gsr_cursor cursor;
-    bool clear_next_frame;
 } gsr_capture_xcomposite;
 
 void gsr_capture_xcomposite_init(gsr_capture_xcomposite *self, const gsr_capture_xcomposite_params *params);

include/egl.h

@@ -110,6 +110,7 @@ typedef void(*__GLXextFuncPtr)(void);
 #define GL_SRC_ALPHA                            0x0302
 #define GL_ONE_MINUS_SRC_ALPHA                  0x0303
 #define GL_DEBUG_OUTPUT                         0x92E0
+#define GL_SCISSOR_TEST                         0x0C11
 
 #define GL_VENDOR                               0x1F00
 #define GL_RENDERER                             0x1F01
@@ -270,11 +271,13 @@ struct gsr_egl {
     void (*glEnableVertexAttribArray)(unsigned int index);
     void (*glDrawArrays)(unsigned int mode, int first, int count);
     void (*glEnable)(unsigned int cap);
+    void (*glDisable)(unsigned int cap);
     void (*glBlendFunc)(unsigned int sfactor, unsigned int dfactor);
     int (*glGetUniformLocation)(unsigned int program, const char *name);
     void (*glUniform1f)(int location, float v0);
     void (*glUniform2f)(int location, float v0, float v1);
     void (*glDebugMessageCallback)(GLDEBUGPROC callback, const void *userParam);
+    void (*glScissor)(int x, int y, int width, int height);
 };
 
 bool gsr_egl_load(gsr_egl *self, Display *dpy, bool wayland, bool is_monitor_capture);

src/capture/kms.c

@@ -358,10 +358,15 @@ bool gsr_capture_kms_capture(gsr_capture_kms *self, AVFrame *frame, bool hdr, bo
         self->base.egl->eglDestroyImage(self->base.egl->egl_display, cursor_image);
         self->base.egl->glBindTexture(target, 0);
 
+        self->base.egl->glEnable(GL_SCISSOR_TEST);
+        self->base.egl->glScissor(target_x, target_y, self->capture_size.x, self->capture_size.y);
+
         gsr_color_conversion_draw(&self->base.color_conversion, self->base.cursor_texture,
             cursor_pos, cursor_size,
             (vec2i){0, 0}, cursor_size,
             texture_rotation, cursor_texture_is_external);
+
+        self->base.egl->glDisable(GL_SCISSOR_TEST);
     }
 
     self->base.egl->eglSwapBuffers(self->base.egl->egl_display, self->base.egl->egl_surface);

src/capture/xcomposite.c

@@ -116,7 +116,6 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     frame->height = video_codec_context->height;
 
     self->window_resize_timer = clock_get_monotonic_seconds();
-    self->clear_next_frame = true;
     return 0;
 }
 
@@ -251,41 +250,27 @@ int gsr_capture_xcomposite_capture(gsr_capture_xcomposite *self, AVFrame *frame)
         target_y + self->cursor.position.y - self->cursor.hotspot.y
     };
 
-    const bool cursor_completely_inside_window =
-        cursor_pos.x >= target_x &&
-        cursor_pos.x + self->cursor.size.x <= target_x + self->texture_size.x &&
-        cursor_pos.y >= target_y &&
-        cursor_pos.y + self->cursor.size.y <= target_y + self->texture_size.y;
-
     const bool cursor_inside_window =
         cursor_pos.x + self->cursor.size.x >= target_x &&
         cursor_pos.x <= target_x + self->texture_size.x &&
         cursor_pos.y + self->cursor.size.y >= target_y &&
         cursor_pos.y <= target_y + self->texture_size.y;
 
-    if(self->clear_next_frame) {
-        self->clear_next_frame = false;
-        gsr_color_conversion_clear(&self->base.color_conversion);
-    }
-
-    /*
-        We dont draw the cursor if it's outside the window but if it's partially inside the window then the cursor area that is outside the window
-        will not get overdrawn the next frame causing a cursor trail to be visible since we dont clear the background.
-        To fix this we detect if the cursor is partially inside the window and clear the background only in that case.
-    */
-    if(!cursor_completely_inside_window && cursor_inside_window && self->params.record_cursor)
-        self->clear_next_frame = true;
-
     gsr_color_conversion_draw(&self->base.color_conversion, window_texture_get_opengl_texture_id(&self->window_texture),
         (vec2i){target_x, target_y}, self->texture_size,
         (vec2i){0, 0}, self->texture_size,
         0.0f, false);
 
     if(cursor_inside_window && self->params.record_cursor) {
+        self->base.egl->glEnable(GL_SCISSOR_TEST);
+        self->base.egl->glScissor(target_x, target_y, self->texture_size.x, self->texture_size.y);
+
         gsr_color_conversion_draw(&self->base.color_conversion, self->cursor.texture_id,
             cursor_pos, self->cursor.size,
             (vec2i){0, 0}, self->cursor.size,
             0.0f, false);
+
+        self->base.egl->glDisable(GL_SCISSOR_TEST);
     }
 
     self->params.egl->eglSwapBuffers(self->params.egl->egl_display, self->params.egl->egl_surface);

src/egl.c

@@ -452,11 +452,13 @@ static bool gsr_egl_load_gl(gsr_egl *self, void *library) {
         { (void**)&self->glEnableVertexAttribArray, "glEnableVertexAttribArray" },
         { (void**)&self->glDrawArrays, "glDrawArrays" },
         { (void**)&self->glEnable, "glEnable" },
+        { (void**)&self->glDisable, "glDisable" },
         { (void**)&self->glBlendFunc, "glBlendFunc" },
         { (void**)&self->glGetUniformLocation, "glGetUniformLocation" },
         { (void**)&self->glUniform1f, "glUniform1f" },
         { (void**)&self->glUniform2f, "glUniform2f" },
         { (void**)&self->glDebugMessageCallback, "glDebugMessageCallback" },
+        { (void**)&self->glScissor, "glScissor" },
 
         { NULL, NULL }
     };

src/main.cpp

@@ -1504,6 +1504,7 @@ static gsr_capture* create_capture_impl(const char *window_str, const char *scre
                     window_str = first_output.output_name;
                 } else {
                     fprintf(stderr, "Error: no available output found\n");
+                    _exit(1);
                 }
             }