nvfbc: mitigate x11 display leak on monitor off

When the monitor is turned off gsr will attempt to recreate the nvfbc
session once a second. If that fails (because the monitor is still
turned off) then nvfbc will leak an x11 display. This seems to be a bug
in the nvidia driver.

Mitigate this by only attempting to recreate the nvfbc session if the
capture target monitor can be found (predicting if nvfbc recreate will
fail).

Thanks to Lim Ding Wen for finding the issue and suggesting the
mitigation.

TODO

@@ -356,3 +356,8 @@ Support youtube sso.
 Remove -fm content (support it but remove it from documentation and output deprecation notice when its used) and use it when using -fm vbr (which is the default option).
     But first -fm content needs to be support on wayland as well, by checking if there is a difference between frames (checksum the frame content).
     -fm content also needs to have a minimum fps to prevent live stream from timing out when nothing changes on the screen.
+
+There is a leak in nvfbc. When a monitor is turned off and then on there will be an x11 display leak inside nvfbc. This seems to be a bug in nvfbc.
+    Right now a mitigation has been added to not try to recreate the nvfbc session if the capture target (monitor) isn't connected (predict if nvfbc session create will fail).
+    One possible reason this happens is because bExternallyManagedContext is set to true.
+    This also means that nvfbc leaks connection when destroying nvfbc, even if the monitor is connected (this is not an issue right now because exit is done, but if gsr was turned into a library it would be).

src/capture/nvfbc.c

@@ -13,6 +13,7 @@
 #include <assert.h>
 
 #include <X11/Xlib.h>
+#include <X11/extensions/Xrandr.h>
 
 typedef struct {
     gsr_capture_nvfbc_params params;
@@ -302,6 +303,35 @@ static int gsr_capture_nvfbc_start(gsr_capture *cap, gsr_capture_metadata *captu
     return -1;
 }
 
+static bool gsr_capture_nvfbc_is_capture_monitor_connected(gsr_capture_nvfbc *self) {
+    Display *dpy = gsr_window_get_display(self->params.egl->window);
+    int num_monitors = 0;
+    XRRMonitorInfo *monitors = XRRGetMonitors(dpy, DefaultRootWindow(dpy), True, &num_monitors);
+    if(!monitors)
+        return false;
+
+    bool capture_monitor_connected = false;
+    if(strcmp(self->params.display_to_capture, "screen") == 0) {
+        capture_monitor_connected = num_monitors > 0;
+    } else {
+        for(int i = 0; i < num_monitors; ++i) {
+            char *monitor_name = XGetAtomName(dpy, monitors[i].name);
+            if(!monitor_name)
+                continue;
+
+            if(strcmp(monitor_name, self->params.display_to_capture) == 0) {
+                capture_monitor_connected = true;
+                XFree(monitor_name);
+                break;
+            }
+            XFree(monitor_name);
+        }
+    }
+
+    XRRFreeMonitors(monitors);
+    return capture_monitor_connected;
+}
+
 static int gsr_capture_nvfbc_capture(gsr_capture *cap, gsr_capture_metadata *capture_metadata, gsr_color_conversion *color_conversion) {
     gsr_capture_nvfbc *self = cap->priv;
 
@@ -310,6 +340,13 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, gsr_capture_metadata *cap
         const double now = clock_get_monotonic_seconds();
         if(now - self->nvfbc_dead_start >= nvfbc_recreate_retry_time_seconds) {
             self->nvfbc_dead_start = now;
+            /*
+                Do not attempt to recreate the nvfbc session if the monitor isn't turned on/connected.
+                This is to predict if the nvfbc session create below will fail since if it fails it leaks an x11 display (a bug in the nvidia driver).
+            */
+            if(!gsr_capture_nvfbc_is_capture_monitor_connected(self))
+                return 0;
+
             gsr_capture_nvfbc_destroy_session_and_handle(self);
 
             if(gsr_capture_nvfbc_setup_handle(self) != 0) {
@@ -322,6 +359,7 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, gsr_capture_metadata *cap
                 return -1;
             }
 
+            fprintf(stderr, "gsr info: gsr_capture_nvfbc_capture: recreated nvfbc session after modeset recovery\n");
             self->nvfbc_needs_recreate = false;
         } else {
             return 0;