Draw cursor in window capture

README.md

@@ -52,7 +52,7 @@ If you install GPU Screen Recorder flatpak, which is the gtk gui version then yo
 libglvnd (which provides libgl and libegl)\
 mesa\
 ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\
-x11 (libx11, libxcomposite, libxrandr)\
+x11 (libx11, libxcomposite, libxrandr, xfixes)\
 libpulse\
 vaapi (libva, libva-mesa-driver)\
 libdrm\
@@ -62,7 +62,7 @@ wayland-client
 libglvnd (which provides libgl and libegl)\
 mesa\
 ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\
-x11 (libx11, libxcomposite, libxrandr)\
+x11 (libx11, libxcomposite, libxrandr, xfixes)\
 libpulse\
 vaapi (libva, libva-intel-driver)\
 libdrm\
@@ -71,7 +71,7 @@ wayland-client
 ## NVIDIA
 libglvnd (which provides libgl and libegl)\
 ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\
-x11 (libx11, libxcomposite, libxrandr)\
+x11 (libx11, libxcomposite, libxrandr, xfixes)\
 libpulse\
 cuda runtime (libcuda.so.1) (libnvidia-compute)\
 nvenc (libnvidia-encode)\
@@ -163,6 +163,5 @@ If you want to donate you can donate via bitcoin or monero.
 
 # TODO
 * Dynamically change bitrate/resolution to match desired fps. This would be helpful when streaming for example, where the encode output speed also depends on upload speed to the streaming service.
-* Show cursor when recording a window. Currently the cursor is only visible when recording a monitor.
 * Implement opengl injection to capture texture. This fixes VRR without having to use NvFBC direct capture.
 * Always use direct capture with NvFBC once the capture issue in mpv fullscreen has been resolved (maybe detect if direct capture fails in nvfbc and switch to non-direct recording. NvFBC says if direct capture fails).

TODO

@@ -118,8 +118,6 @@ Use CAP_SYS_NICE in flatpak too on the main gpu screen recorder binary. It makes
 
 Show error when using compressed kms plane which isn't supported. Also do that in the gui.
 
-Use video_codec_context->width/height instead of frame->width/height in capture.
-
 Modify ffmpeg to accept opengl texture for nvenc encoding. Removes extra buffers and copies.
 
 When vulkan encode is added, mention minimum nvidia driver required. (550.54.14?).

build.sh

@@ -20,7 +20,7 @@ build_gsr_kms_server() {
 }
 
 build_gsr() {
-    dependencies="libavcodec libavformat libavutil x11 xcomposite xrandr libpulse libswresample libavfilter libva libcap libdrm wayland-egl wayland-client"
+    dependencies="libavcodec libavformat libavutil x11 xcomposite xrandr xfixes libpulse libswresample libavfilter libva libcap libdrm wayland-egl wayland-client"
     includes="$(pkg-config --cflags $dependencies)"
     libs="$(pkg-config --libs $dependencies) -ldl -pthread -lm"
     $CC -c src/capture/capture.c $opts $includes
@@ -41,10 +41,11 @@ build_gsr() {
     $CC -c src/color_conversion.c $opts $includes
     $CC -c src/utils.c $opts $includes
     $CC -c src/library_loader.c $opts $includes
+    $CC -c src/cursor.c $opts $includes
     $CXX -c src/sound.cpp $opts $includes
     $CXX -c src/main.cpp $opts $includes
     $CXX -o gpu-screen-recorder capture.o nvfbc.o kms_client.o egl.o cuda.o xnvctrl.o overclock.o window_texture.o shader.o \
-        color_conversion.o utils.o library_loader.o xcomposite.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o kms.o sound.o main.o $libs $opts
+        color_conversion.o utils.o library_loader.o cursor.o xcomposite.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o kms.o sound.o main.o $libs $opts
 }
 
 build_gsr_kms_server

include/capture/xcomposite.h

@@ -6,6 +6,7 @@
 #include "../vec2.h"
 #include "../color_conversion.h"
 #include "../window_texture.h"
+#include "../cursor.h"
 
 typedef struct {
     gsr_egl *egl;
@@ -33,6 +34,9 @@ typedef struct {
     WindowTexture window_texture;
 
     Atom net_active_window_atom;
+
+    gsr_cursor cursor;
+    bool clear_next_frame;
 } gsr_capture_xcomposite;
 
 void gsr_capture_xcomposite_init(gsr_capture_xcomposite *self, const gsr_capture_xcomposite_params *params);

include/cursor.h

@@ -0,0 +1,26 @@
+#ifndef GSR_CURSOR_H
+#define GSR_CURSOR_H
+
+#include "egl.h"
+#include "vec2.h"
+
+typedef struct {
+    gsr_egl *egl;
+    Display *display;
+    int x_fixes_event_base;
+
+    unsigned int texture_id;
+    vec2i size;
+    vec2i hotspot;
+    vec2i position;
+
+    bool cursor_image_set;
+} gsr_cursor;
+
+int gsr_cursor_init(gsr_cursor *self, gsr_egl *egl, Display *display);
+void gsr_cursor_deinit(gsr_cursor *self);
+
+void gsr_cursor_update(gsr_cursor *self, XEvent *xev);
+void gsr_cursor_tick(gsr_cursor *self, Window relative_to);
+
+#endif /* GSR_CURSOR_H */

project.conf

@@ -15,6 +15,7 @@ libavutil = ">=56.2"
 x11 = ">=1"
 xcomposite = ">=0.2"
 xrandr = ">=1"
+xfixes = ">=2"
 libpulse = ">=13"
 libswresample = ">=3"
 libavfilter = ">=5"

src/capture/nvfbc.c

@@ -397,8 +397,8 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame) {
     cap_nvfbc->params.egl->glClear(0);
 
     gsr_color_conversion_draw(&cap_nvfbc->base.color_conversion, cap_nvfbc->setup_params.dwTextures[grab_params.dwTextureIndex],
-        (vec2i){0, 0}, (vec2i){cap_nvfbc->base.video_codec_context->width, cap_nvfbc->base.video_codec_context->height},
-        (vec2i){0, 0}, (vec2i){cap_nvfbc->base.video_codec_context->width, cap_nvfbc->base.video_codec_context->height},
+        (vec2i){0, 0}, (vec2i){frame->width, frame->height},
+        (vec2i){0, 0}, (vec2i){frame->width, frame->height},
         0.0f, false);
 
     cap_nvfbc->params.egl->glXSwapBuffers(cap_nvfbc->params.egl->x11.dpy, cap_nvfbc->params.egl->x11.window);
@@ -416,11 +416,11 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame) {
         memcpy_struct.dstMemoryType = CU_MEMORYTYPE_DEVICE;
 
         memcpy_struct.srcArray = cap_nvfbc->mapped_arrays[i];
-        memcpy_struct.srcPitch = cap_nvfbc->base.video_codec_context->width / div[i];
+        memcpy_struct.srcPitch = frame->width / div[i];
         memcpy_struct.dstDevice = (CUdeviceptr)frame->data[i];
         memcpy_struct.dstPitch = frame->linesize[i];
-        memcpy_struct.WidthInBytes = cap_nvfbc->base.video_codec_context->width * (cap_nvfbc->params.hdr ? 2 : 1);
-        memcpy_struct.Height = cap_nvfbc->base.video_codec_context->height / div[i];
+        memcpy_struct.WidthInBytes = frame->width * (cap_nvfbc->params.hdr ? 2 : 1);
+        memcpy_struct.Height = frame->height / div[i];
         // TODO: Remove this copy if possible
         cap_nvfbc->cuda.cuMemcpy2DAsync_v2(&memcpy_struct, cap_nvfbc->cuda_stream);
     }

src/capture/xcomposite.c

@@ -57,8 +57,8 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     /* TODO: Do these in tick, and allow error if follow_focused */
 
     XWindowAttributes attr;
-    if(!XGetWindowAttributes(self->params.egl->x11.dpy, self->params.window, &attr) && !self->params.follow_focused) {
-        fprintf(stderr, "gsr error: gsr_capture_xcomposite_start failed: invalid window id: %lu\n", self->params.window);
+    if(!XGetWindowAttributes(self->params.egl->x11.dpy, self->window, &attr) && !self->params.follow_focused) {
+        fprintf(stderr, "gsr error: gsr_capture_xcomposite_start failed: invalid window id: %lu\n", self->window);
         return -1;
     }
 
@@ -69,7 +69,7 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
         XSelectInput(self->params.egl->x11.dpy, DefaultRootWindow(self->params.egl->x11.dpy), PropertyChangeMask);
 
     // TODO: Get select and add these on top of it and then restore at the end. Also do the same in other xcomposite
-    XSelectInput(self->params.egl->x11.dpy, self->params.window, StructureNotifyMask | ExposureMask);
+    XSelectInput(self->params.egl->x11.dpy, self->window, StructureNotifyMask | ExposureMask);
 
     if(!self->params.egl->eglExportDMABUFImageQueryMESA) {
         fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: could not find eglExportDMABUFImageQueryMESA\n");
@@ -83,8 +83,13 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
 
     /* Disable vsync */
     self->params.egl->eglSwapInterval(self->params.egl->egl_display, 0);
-    if(window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->params.window, self->params.egl) != 0 && !self->params.follow_focused) {
-        fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: failed to get window texture for window %ld\n", self->params.window);
+    if(window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->window, self->params.egl) != 0 && !self->params.follow_focused) {
+        fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: failed to get window texture for window %ld\n", self->window);
+        return -1;
+    }
+
+    if(gsr_cursor_init(&self->cursor, self->params.egl, self->params.egl->x11.dpy) != 0) {
+        gsr_capture_xcomposite_stop(self, video_codec_context);
         return -1;
     }
 
@@ -115,14 +120,9 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
 }
 
 void gsr_capture_xcomposite_stop(gsr_capture_xcomposite *self, AVCodecContext *video_codec_context) {
+    (void)video_codec_context;
     window_texture_deinit(&self->window_texture);
-
-    if(video_codec_context->hw_device_ctx)
-        av_buffer_unref(&video_codec_context->hw_device_ctx);
-    if(video_codec_context->hw_frames_ctx)
-        av_buffer_unref(&video_codec_context->hw_frames_ctx);
-
-    gsr_capture_base_stop(&self->base, self->params.egl);
+    gsr_cursor_deinit(&self->cursor);
 }
 
 void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *video_codec_context) {
@@ -168,6 +168,8 @@ void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *v
                 break;
             }
         }
+
+        gsr_cursor_update(&self->cursor, &self->xev);
     }
 
     if(self->params.follow_focused && !self->follow_focused_initialized) {
@@ -194,17 +196,6 @@ void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *v
 
             window_texture_deinit(&self->window_texture);
             window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->window, self->params.egl); // TODO: Do not do the below window_texture_on_resize after this
-            
-            self->texture_size.x = 0;
-            self->texture_size.y = 0;
-
-            self->params.egl->glBindTexture(GL_TEXTURE_2D, window_texture_get_opengl_texture_id(&self->window_texture));
-            self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &self->texture_size.x);
-            self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &self->texture_size.y);
-            self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
-
-            self->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(self->texture_size.x)));
-            self->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(self->texture_size.y)));
         }
     }
 
@@ -252,11 +243,51 @@ int gsr_capture_xcomposite_capture(gsr_capture_xcomposite *self, AVFrame *frame)
     const int target_x = max_int(0, frame->width / 2 - self->texture_size.x / 2);
     const int target_y = max_int(0, frame->height / 2 - self->texture_size.y / 2);
 
+    // TODO: Can we do this a better way than to call it every capture?
+    gsr_cursor_tick(&self->cursor, self->window);
+
+    const vec2i cursor_pos = {
+        target_x + self->cursor.position.x - self->cursor.hotspot.x,
+        target_y + self->cursor.position.y - self->cursor.hotspot.y
+    };
+
+    const bool cursor_completely_inside_window =
+        cursor_pos.x >= target_x &&
+        cursor_pos.x <= target_x + self->texture_size.x &&
+        cursor_pos.y >= target_y &&
+        cursor_pos.y <= target_y + self->texture_size.x;
+
+    const bool cursor_inside_window =
+        cursor_pos.x + self->cursor.size.x >= target_x &&
+        cursor_pos.x <= target_x + self->texture_size.x &&
+        cursor_pos.y + self->cursor.size.y >= target_y &&
+        cursor_pos.y <= target_y + self->texture_size.x;
+
+    if(self->clear_next_frame) {
+        self->clear_next_frame = false;
+        gsr_color_conversion_clear(&self->base.color_conversion);
+    }
+
+    /*
+        We dont draw the cursor if it's outside the window but if it's partially inside the window then the cursor area that is outside the window
+        will not get overdrawn the next frame causing a cursor trail to be visible since we dont clear the background.
+        To fix this we detect if the cursor is partially inside the window and clear the background only in that case.
+    */
+    if(!cursor_completely_inside_window && cursor_inside_window)
+        self->clear_next_frame = true;
+
     gsr_color_conversion_draw(&self->base.color_conversion, window_texture_get_opengl_texture_id(&self->window_texture),
         (vec2i){target_x, target_y}, self->texture_size,
         (vec2i){0, 0}, self->texture_size,
         0.0f, false);
 
+    if(cursor_inside_window) {
+        gsr_color_conversion_draw(&self->base.color_conversion, self->cursor.texture_id,
+            cursor_pos, self->cursor.size,
+            (vec2i){0, 0}, self->cursor.size,
+            0.0f, false);
+    }
+
     self->params.egl->eglSwapBuffers(self->params.egl->egl_display, self->params.egl->egl_surface);
     //self->params.egl->glFlush();
     //self->params.egl->glFinish();

src/capture/xcomposite_cuda.c

@@ -3,6 +3,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <libavutil/frame.h>
+#include <libavcodec/avcodec.h>
 
 typedef struct {
     gsr_capture_xcomposite xcomposite;
@@ -70,6 +71,13 @@ static void gsr_capture_xcomposite_unload_cuda_graphics(gsr_capture_xcomposite_c
 
 static void gsr_capture_xcomposite_cuda_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
     gsr_capture_xcomposite_cuda *cap_xcomp = cap->priv;
+
+    if(video_codec_context->hw_device_ctx)
+        av_buffer_unref(&video_codec_context->hw_device_ctx);
+    if(video_codec_context->hw_frames_ctx)
+        av_buffer_unref(&video_codec_context->hw_frames_ctx);
+
+    gsr_capture_base_stop(&cap_xcomp->xcomposite.base, cap_xcomp->xcomposite.params.egl);
     gsr_capture_xcomposite_stop(&cap_xcomp->xcomposite, video_codec_context);
     gsr_capture_xcomposite_unload_cuda_graphics(cap_xcomp);
     gsr_cuda_unload(&cap_xcomp->cuda);

src/capture/xcomposite_vaapi.c

@@ -5,6 +5,7 @@
 #include <stdio.h>
 #include <va/va.h>
 #include <va/va_drmcommon.h>
+#include <libavcodec/avcodec.h>
 
 typedef struct {
     gsr_capture_xcomposite xcomposite;
@@ -62,6 +63,12 @@ static void gsr_capture_xcomposite_vaapi_stop(gsr_capture *cap, AVCodecContext *
         }
     }
 
+    if(video_codec_context->hw_device_ctx)
+        av_buffer_unref(&video_codec_context->hw_device_ctx);
+    if(video_codec_context->hw_frames_ctx)
+        av_buffer_unref(&video_codec_context->hw_frames_ctx);
+
+    gsr_capture_base_stop(&cap_xcomp->xcomposite.base, cap_xcomp->xcomposite.params.egl);
     gsr_capture_xcomposite_stop(&cap_xcomp->xcomposite, video_codec_context);
 }
 

src/cursor.c

@@ -0,0 +1,127 @@
+#include "../include/cursor.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include <X11/extensions/Xfixes.h>
+
+static bool gsr_cursor_set_from_x11_cursor_image(gsr_cursor *self, XFixesCursorImage *x11_cursor_image) {
+    uint8_t *cursor_data = NULL;
+    uint8_t *out = NULL;
+
+    if(!x11_cursor_image)
+        goto err;
+        
+    if(!x11_cursor_image->pixels)
+        goto err;
+
+    self->hotspot.x = x11_cursor_image->xhot;
+    self->hotspot.y = x11_cursor_image->yhot;
+    self->egl->glBindTexture(GL_TEXTURE_2D, self->texture_id);
+
+    self->size.x = x11_cursor_image->width;
+    self->size.y = x11_cursor_image->height;
+    const unsigned long *pixels = x11_cursor_image->pixels;
+    cursor_data = malloc(self->size.x * self->size.y * 4);
+    if(!cursor_data)
+        goto err;
+    out = cursor_data;
+    /* Un-premultiply alpha */
+    for(int y = 0; y < self->size.y; ++y) {
+        for(int x = 0; x < self->size.x; ++x) {
+            uint32_t pixel = *pixels++;
+            uint8_t *in = (uint8_t*)&pixel;
+            uint8_t alpha = in[3];
+            if(alpha == 0)
+                alpha = 1;
+
+            *out++ = (unsigned)*in++ * 255/alpha;
+            *out++ = (unsigned)*in++ * 255/alpha;
+            *out++ = (unsigned)*in++ * 255/alpha;
+            *out++ = *in++;
+        }
+    }
+
+    self->egl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, self->size.x, self->size.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, cursor_data);
+    free(cursor_data);
+
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+
+    self->egl->glBindTexture(GL_TEXTURE_2D, 0);
+    XFree(x11_cursor_image);
+    return true;
+
+    err:
+    self->egl->glBindTexture(GL_TEXTURE_2D, 0);
+    if(x11_cursor_image)
+        XFree(x11_cursor_image);
+    return false;
+}
+
+int gsr_cursor_init(gsr_cursor *self, gsr_egl *egl, Display *display) {
+    int x_fixes_error_base = 0;
+
+    assert(egl);
+    assert(display);
+    memset(self, 0, sizeof(*self));
+    self->egl = egl;
+    self->display = display;
+
+    self->x_fixes_event_base = 0;
+    if(!XFixesQueryExtension(self->display, &self->x_fixes_event_base, &x_fixes_error_base)) {
+        fprintf(stderr, "gsr error: gsr_cursor_init: your X11 server is missing the XFixes extension\n");
+        gsr_cursor_deinit(self);
+        return -1;
+    }
+
+    self->egl->glGenTextures(1, &self->texture_id);
+
+    XFixesSelectCursorInput(self->display, DefaultRootWindow(self->display), XFixesDisplayCursorNotifyMask);
+    gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display));
+    self->cursor_image_set = true;
+
+    return 0;
+}
+
+void gsr_cursor_deinit(gsr_cursor *self) {
+    if(!self->egl)
+        return;
+
+    if(self->texture_id) {
+        self->egl->glDeleteTextures(1, &self->texture_id);
+        self->texture_id = 0;
+    }
+
+    XFixesSelectCursorInput(self->display, DefaultRootWindow(self->display), 0);
+
+    self->display = NULL;
+    self->egl = NULL;
+}
+
+void gsr_cursor_update(gsr_cursor *self, XEvent *xev) {
+    if(xev->type == self->x_fixes_event_base + XFixesCursorNotify) {
+        XFixesCursorNotifyEvent *cursor_notify_event = (XFixesCursorNotifyEvent*)xev;
+        if(cursor_notify_event->subtype == XFixesDisplayCursorNotify && cursor_notify_event->window == DefaultRootWindow(self->display)) {
+            self->cursor_image_set = true;
+            gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display));
+        }
+    }
+
+    if(!self->cursor_image_set) {
+        self->cursor_image_set = true;
+        gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display));
+    }
+}
+
+void gsr_cursor_tick(gsr_cursor *self, Window relative_to) {
+    /* TODO: Use XInput2 instead. However that doesn't work when the pointer is grabbed. Maybe check for focused window change and XSelectInput PointerMask */
+    Window dummy_window;
+    int dummy_i;
+    unsigned int dummy_u;
+    XQueryPointer(self->display, relative_to, &dummy_window, &dummy_window, &dummy_i, &dummy_i, &self->position.x, &self->position.y, &dummy_u);
+}