Move damage tracking to separate file

TODO

@@ -157,3 +157,5 @@ Allow prime-run on x11 if monitor capture and the prime gpu is not nvidia.
 Enable 2-pass encoding.
 
 Add vbr/cbr option.
+
+Restart replay/update video resolution if monitor resolution changes.

include/damage.h

@@ -0,0 +1,27 @@
+#ifndef GSR_DAMAGE_H
+#define GSR_DAMAGE_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef struct _XDisplay Display;
+typedef union _XEvent XEvent;
+
+typedef struct {
+    Display *display;
+    int damage_event;
+    int damage_error;
+    uint64_t damage;
+    bool damaged;
+} gsr_damage;
+
+bool gsr_damage_init(gsr_damage *self, Display *display);
+void gsr_damage_deinit(gsr_damage *self);
+
+bool gsr_damage_set_target_window(gsr_damage *self, uint64_t window);
+void gsr_damage_update(gsr_damage *self, XEvent *xev);
+/* Also returns true if damage tracking is not available */
+bool gsr_damage_is_damaged(gsr_damage *self);
+void gsr_damage_clear(gsr_damage *self);
+
+#endif /* GSR_DAMAGE_H */

meson.build

@@ -27,6 +27,7 @@ src = [
     'src/utils.c',
     'src/library_loader.c',
     'src/cursor.c',
+    'src/damage.c',
     'src/sound.cpp',
     'src/main.cpp',
 ]

src/capture/xcomposite.c

@@ -2,6 +2,7 @@
 #include "../../include/window_texture.h"
 #include "../../include/utils.h"
 #include "../../include/cursor.h"
+#include "../../include/damage.h"
 #include "../../include/color_conversion.h"
 
 #include <stdlib.h>
@@ -10,7 +11,6 @@
 #include <assert.h>
 
 #include <X11/Xlib.h>
-#include <X11/extensions/Xdamage.h>
 
 #include <libavutil/frame.h>
 #include <libavcodec/avcodec.h>
@@ -34,23 +34,16 @@ typedef struct {
     Atom net_active_window_atom;
 
     gsr_cursor cursor;
-
-    int damage_event;
-    int damage_error;
-    XID damage;
-    bool damaged;
+    gsr_damage damage;
+    bool cursor_damaged;
 
     bool clear_background;
 } gsr_capture_xcomposite;
 
 static void gsr_capture_xcomposite_stop(gsr_capture_xcomposite *self) {
-    if(self->damage) {
-        XDamageDestroy(self->params.egl->x11.dpy, self->damage);
-        self->damage = None;
-    }
-
     window_texture_deinit(&self->window_texture);
     gsr_cursor_deinit(&self->cursor);
+    gsr_damage_deinit(&self->damage);
 }
 
 static int max_int(int a, int b) {
@@ -71,23 +64,6 @@ static Window get_focused_window(Display *display, Atom net_active_window_atom)
     return None;
 }
 
-static void gsr_capture_xcomposite_setup_damage(gsr_capture_xcomposite *self, Window window) {
-    if(self->damage_event == 0)
-        return;
-
-    if(self->damage) {
-        XDamageDestroy(self->params.egl->x11.dpy, self->damage);
-        self->damage = None;
-    }
-
-    self->damage = XDamageCreate(self->params.egl->x11.dpy, window, XDamageReportNonEmpty);
-    if(self->damage) {
-        XDamageSubtract(self->params.egl->x11.dpy, self->damage, None, None);
-    } else {
-        fprintf(stderr, "gsr warning: gsr_capture_xcomposite_setup_damage: XDamageCreate failed\n");
-    }
-}
-
 static int gsr_capture_xcomposite_start(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame *frame) {
     gsr_capture_xcomposite *self = cap->priv;
 
@@ -102,19 +78,12 @@ static int gsr_capture_xcomposite_start(gsr_capture *cap, AVCodecContext *video_
         self->window = self->params.window;
     }
 
-    if(self->params.track_damage) {
-        if(!XDamageQueryExtension(self->params.egl->x11.dpy, &self->damage_event, &self->damage_error)) {
-            fprintf(stderr, "gsr warning: gsr_capture_xcomposite_start: XDamage is not supported by your X11 server\n");
-            self->damage_event = 0;
-            self->damage_error = 0;
-        }
-    } else {
-        self->damage_event = 0;
-        self->damage_error = 0;
-    }
+    if(self->params.track_damage)
+        gsr_damage_init(&self->damage, self->params.egl->x11.dpy);
+    else
+        memset(&self->damage, 0, sizeof(self->damage));
 
-    self->damaged = true;
-    gsr_capture_xcomposite_setup_damage(self, self->window);
+    gsr_damage_set_target_window(&self->damage, self->window);
 
     /* TODO: Do these in tick, and allow error if follow_focused */
 
@@ -212,18 +181,10 @@ static void gsr_capture_xcomposite_tick(gsr_capture *cap, AVCodecContext *video_
             }
         }
 
-        if(self->damage_event && self->xev.type == self->damage_event + XDamageNotify) {
-            XDamageNotifyEvent *de = (XDamageNotifyEvent*)&self->xev;
-            XserverRegion region = XFixesCreateRegion(self->params.egl->x11.dpy, NULL, 0);
-            // Subtract all the damage, repairing the window
-            XDamageSubtract(self->params.egl->x11.dpy, de->damage, None, region);
-            XFixesDestroyRegion(self->params.egl->x11.dpy, region);
-            self->damaged = true;
-        }
-
+        gsr_damage_update(&self->damage, &self->xev);
         if(gsr_cursor_update(&self->cursor, &self->xev)) {
             if(self->params.record_cursor && self->cursor.visible) {
-                self->damaged = true;
+                self->cursor_damaged = true;
             }
         }
     }
@@ -262,7 +223,7 @@ static void gsr_capture_xcomposite_tick(gsr_capture *cap, AVCodecContext *video_
 
             self->window_resized = false;
             self->clear_background = true;
-            gsr_capture_xcomposite_setup_damage(self, self->window);
+            gsr_damage_set_target_window(&self->damage, self->window);
         }
     }
 
@@ -286,18 +247,19 @@ static void gsr_capture_xcomposite_tick(gsr_capture *cap, AVCodecContext *video_
         self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
         self->clear_background = true;
-        gsr_capture_xcomposite_setup_damage(self, self->window);
+        gsr_damage_set_target_window(&self->damage, self->window);
     }
 }
 
 static bool gsr_capture_xcomposite_is_damaged(gsr_capture *cap) {
     gsr_capture_xcomposite *self = cap->priv;
-    return self->damage_event ? self->damaged : true;
+    return gsr_damage_is_damaged(&self->damage) || self->cursor_damaged;
 }
 
 static void gsr_capture_xcomposite_clear_damage(gsr_capture *cap) {
     gsr_capture_xcomposite *self = cap->priv;
-    self->damaged = false;
+    gsr_damage_clear(&self->damage);
+    self->cursor_damaged = false;
 }
 
 static bool gsr_capture_xcomposite_should_stop(gsr_capture *cap, bool *err) {

src/damage.c

@@ -0,0 +1,70 @@
+#include "../include/damage.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <X11/extensions/Xdamage.h>
+
+bool gsr_damage_init(gsr_damage *self, Display *display) {
+    memset(self, 0, sizeof(*self));
+    self->display = display;
+
+    if(!XDamageQueryExtension(self->display, &self->damage_event, &self->damage_error)) {
+        fprintf(stderr, "gsr warning: gsr_damage_init: XDamage is not supported by your X11 server\n");
+        self->damage_event = 0;
+        self->damage_error = 0;
+        return false;
+    }
+
+    self->damaged = true;
+    return true;
+}
+
+void gsr_damage_deinit(gsr_damage *self) {
+    if(self->damage) {
+        XDamageDestroy(self->display, self->damage);
+        self->damage = None;
+    }
+}
+
+bool gsr_damage_set_target_window(gsr_damage *self, uint64_t window) {
+    if(self->damage_event == 0)
+        return false;
+
+    if(self->damage) {
+        XDamageDestroy(self->display, self->damage);
+        self->damage = None;
+    }
+
+    self->damage = XDamageCreate(self->display, window, XDamageReportNonEmpty);
+    if(self->damage) {
+        XDamageSubtract(self->display, self->damage, None, None);
+        self->damaged = true;
+        return true;
+    } else {
+        fprintf(stderr, "gsr warning: gsr_damage_set_target_window: XDamageCreate failed\n");
+        return false;
+    }
+}
+
+void gsr_damage_update(gsr_damage *self, XEvent *xev) {
+    if(self->damage_event == 0 || !self->damage)
+        return;
+
+    if(self->damage_event && xev->type == self->damage_event + XDamageNotify) {
+        XDamageNotifyEvent *de = (XDamageNotifyEvent*)xev;
+        XserverRegion region = XFixesCreateRegion(self->display, NULL, 0);
+        /* Subtract all the damage, repairing the window */
+        XDamageSubtract(self->display, de->damage, None, region);
+        XFixesDestroyRegion(self->display, region);
+        XFlush(self->display);
+        self->damaged = true;
+    }
+}
+
+bool gsr_damage_is_damaged(gsr_damage *self) {
+    return self->damage_event == 0 || !self->damage || self->damaged;
+}
+
+void gsr_damage_clear(gsr_damage *self) {
+    self->damaged = false;
+}

src/pipewire.c

@@ -663,16 +663,14 @@ static bool gsr_pipewire_bind_image_to_texture(gsr_pipewire *self, EGLImage imag
     return success;
 }
 
-static void gsr_pipewire_bind_image_to_texture_with_fallback(gsr_pipewire *self, gsr_texture_map texture_map, EGLImage image, bool *using_external_image) {
+static void gsr_pipewire_bind_image_to_texture_with_fallback(gsr_pipewire *self, gsr_texture_map texture_map, EGLImage image) {
     if(self->external_texture_fallback) {
         gsr_pipewire_bind_image_to_texture(self, image, texture_map.external_texture_id, true);
-        *using_external_image = true;
     } else {
         if(!gsr_pipewire_bind_image_to_texture(self, image, texture_map.texture_id, false)) {
             fprintf(stderr, "gsr error: gsr_pipewire_map_texture: failed to bind image to texture, trying with external texture\n");
             self->external_texture_fallback = true;
             gsr_pipewire_bind_image_to_texture(self, image, texture_map.external_texture_id, true);
-            *using_external_image = true;
         }
     }
 }
@@ -709,7 +707,8 @@ bool gsr_pipewire_map_texture(gsr_pipewire *self, gsr_texture_map texture_map, g
 
     EGLImage image = gsr_pipewire_create_egl_image_with_fallback(self);
     if(image) {
-        gsr_pipewire_bind_image_to_texture_with_fallback(self, texture_map, image, using_external_image);
+        gsr_pipewire_bind_image_to_texture_with_fallback(self, texture_map, image);
+        *using_external_image = self->external_texture_fallback;
         self->egl->eglDestroyImage(self->egl->egl_display, image);
     }