Use X11 global hotkeys on X11 when possible to prevent clashing with keys used by other applications

README.md

@@ -48,4 +48,4 @@ If you want to donate you can donate via bitcoin or monero.
 # Known issues
 * The UI always opens on the same (incorrect) monitor when using multiple monitors on Wayland
 * Some games receive mouse input while the UI is open
-* Global hotkeys can clash with other hotkeys. This is primarly because Wayland compositors are missing support for global hotkey so this software uses a global hotkey system that works on X11 and Wayland by bypassing X11 and Wayland.
+* Global hotkeys on Wayland can clash with keys used by other applications. This is primarly because Wayland compositors are missing support for global hotkey so this software uses a global hotkey system that works on all Wayland compositors.

TODO

@@ -100,3 +100,7 @@ Remove all dependencies from tools/gsr-global-hotkeys and roll our own keyboard
 Test global hotkeys with azerty instead of qwerty.
 
 Fix cursor grab not working in owlboy, need to use xigrab.
+
+Dont allow autostart of replay if capture option is window recording (when window recording is added).
+
+Use global shortcuts desktop portal protocol on wayland when available.

include/GlobalHotkeys.hpp

@@ -4,6 +4,10 @@
 #include <functional>
 #include <string>
 
+namespace mgl {
+    class Event;
+}
+
 namespace gsr {
     struct Hotkey {
         uint64_t key = 0;
@@ -24,5 +28,7 @@ namespace gsr {
         virtual void unbind_all_keys() {}
         virtual bool bind_action(const std::string &id, GlobalHotkeyCallback callback) { (void)id; (void)callback; return false; };
         virtual void poll_events() = 0;
+        // Returns true if the event wasn't consumed (if the event didn't match a key that has been bound)
+        virtual bool on_event(mgl::Event &event) { (void)event; return true; }
     };
 }

include/GlobalHotkeysX11.hpp

@@ -12,12 +12,15 @@ namespace gsr {
         GlobalHotkeysX11& operator=(const GlobalHotkeysX11&) = delete;
         ~GlobalHotkeysX11() override;
 
+        // Hotkey key is a KeySym (XK_z for example) and modifiers is a bitmask of X11 modifier masks (for example ShiftMask | Mod1Mask)
         bool bind_key_press(Hotkey hotkey, const std::string &id, GlobalHotkeyCallback callback) override;
         void unbind_key_press(const std::string &id) override;
         void unbind_all_keys() override;
         void poll_events() override;
+        bool on_event(mgl::Event &event) override;
     private:
-        void call_hotkey_callback(Hotkey hotkey) const;
+        // Returns true if a key bind has been registered for the hotkey
+        bool call_hotkey_callback(Hotkey hotkey) const;
     private:
         struct HotkeyData {
             Hotkey hotkey;

include/Overlay.hpp

@@ -18,6 +18,7 @@
 
 namespace gsr {
     class DropdownButton;
+    class GlobalHotkeys;
 
     enum class RecordingStatus {
         NONE,
@@ -40,7 +41,7 @@ namespace gsr {
         Overlay& operator=(const Overlay&) = delete;
         ~Overlay();
 
-        void handle_events();
+        void handle_events(gsr::GlobalHotkeys *global_hotkeys);
         void on_event(mgl::Event &event);
         // Returns false if not visible
         bool draw();

src/GlobalHotkeysX11.cpp

@@ -1,6 +1,7 @@
 #include "../include/GlobalHotkeysX11.hpp"
-#define XK_MISCELLANY
-#include <X11/keysymdef.h>
+#include <X11/keysym.h>
+#include <mglpp/window/Event.hpp>
+#include <assert.h>
 
 namespace gsr {
     static bool x_failed = false;
@@ -25,6 +26,30 @@ namespace gsr {
         return numlockmask;
     }
 
+    static KeySym mgl_key_to_key_sym(mgl::Keyboard::Key key) {
+        switch(key) {
+            case mgl::Keyboard::Z:   return XK_z;
+            case mgl::Keyboard::F7:  return XK_F7;
+            case mgl::Keyboard::F8:  return XK_F8;
+            case mgl::Keyboard::F9:  return XK_F9;
+            case mgl::Keyboard::F10: return XK_F10;
+            default:                 return None;
+        }
+    }
+
+    static uint32_t mgl_key_modifiers_to_x11_modifier_mask(const mgl::Event::KeyEvent &key_event) {
+        uint32_t mask = 0;
+        if(key_event.shift)
+            mask |= ShiftMask;
+        if(key_event.control)
+            mask |= ControlMask;
+        if(key_event.alt)
+            mask |= Mod1Mask;
+        if(key_event.system)
+            mask |= Mod4Mask;
+        return mask;
+    }
+
     GlobalHotkeysX11::GlobalHotkeysX11() {
         dpy = XOpenDisplay(NULL);
         if(!dpy)
@@ -122,16 +147,27 @@ namespace gsr {
         }
     }
 
+    bool GlobalHotkeysX11::on_event(mgl::Event &event) {
+        if(event.type != mgl::Event::KeyPressed)
+            return true;
+
+        // Note: not all keys are mapped in mgl_key_to_key_sym. If more hotkeys are added or changed then add the key mapping there
+        const KeySym key_sym = mgl_key_to_key_sym(event.key.code);
+        const uint32_t modifiers = mgl_key_modifiers_to_x11_modifier_mask(event.key);
+        return !call_hotkey_callback(Hotkey{key_sym, modifiers});
+    }
+
     static unsigned int key_state_without_locks(unsigned int key_state) {
         return key_state & ~(Mod2Mask|LockMask);
     }
 
-    void GlobalHotkeysX11::call_hotkey_callback(Hotkey hotkey) const {
+    bool GlobalHotkeysX11::call_hotkey_callback(Hotkey hotkey) const {
         for(const auto &[key, val] : bound_keys_by_id) {
             if(val.hotkey.key == hotkey.key && key_state_without_locks(val.hotkey.modifiers) == key_state_without_locks(hotkey.modifiers)) {
                 val.callback(key);
-                return;
+                return true;
             }
         }
+        return false;
     }
 }

src/Overlay.cpp

@@ -10,6 +10,7 @@
 #include "../include/gui/Utils.hpp"
 #include "../include/gui/PageStack.hpp"
 #include "../include/WindowUtils.hpp"
+#include "../include/GlobalHotkeys.hpp"
 
 #include <string.h>
 #include <assert.h>
@@ -555,13 +556,17 @@ namespace gsr {
         }
     }
 
-    void Overlay::handle_events() {
+    void Overlay::handle_events(gsr::GlobalHotkeys *global_hotkeys) {
         if(!visible || !window)
             return;
 
         handle_xi_events();
 
         while(window->poll_event(event)) {
+            if(global_hotkeys) {
+                if(!global_hotkeys->on_event(event))
+                    continue;
+            }
             on_event(event);
         }
     }
@@ -910,7 +915,7 @@ namespace gsr {
         // The real cursor doesn't move when all devices are grabbed, so we create our own cursor and diplay that while grabbed
         xi_setup_fake_cursor();
 
-        // We want to grab all devices to prevent any other application below from receiving events.
+        // We want to grab all devices to prevent any other application below the UI from receiving events.
         // Owlboy seems to use xi events and XGrabPointer doesn't prevent owlboy from receiving events.
         xi_grab_all_devices();
 
@@ -1005,11 +1010,17 @@ namespace gsr {
     }
 
     void Overlay::toggle_show() {
-        if(visible)
-            hide();
-        else
+        if(visible) {
+            //hide();
+            // We dont want to hide immediately because hide is called in event callback, in which it destroys the window.
+            // Instead remove all pages and wait until next iteration to close the UI (which happens when there are no pages to render).
+            while(!page_stack.empty()) {
+                page_stack.pop();
+            }
+        } else {
             show();
         }
+    }
 
     void Overlay::toggle_record() {
         on_press_start_record();
@@ -1555,6 +1566,8 @@ namespace gsr {
         const int fdl = fcntl(gpu_screen_recorder_process_output_fd, F_GETFL);
         fcntl(gpu_screen_recorder_process_output_fd, F_SETFL, fdl | O_NONBLOCK);
         gpu_screen_recorder_process_output_file = fdopen(gpu_screen_recorder_process_output_fd, "r");
+        if(gpu_screen_recorder_process_output_file)
+            gpu_screen_recorder_process_output_fd = -1;
 
         // TODO: Start recording after this notification has disappeared to make sure it doesn't show up in the video.
         // Make clear to the user that the recording starts after the notification is gone.

src/main.cpp

@@ -14,7 +14,7 @@
 #include <mglpp/mglpp.hpp>
 #include <mglpp/system/Clock.hpp>
 
-// TODO: Make keyboard controllable for steam deck (and other controllers).
+// TODO: Make keyboard/controller controllable for steam deck (and other controllers).
 // TODO: Keep track of gpu screen recorder run by other programs to not allow recording at the same time, or something.
 // TODO: Add systray by using org.kde.StatusNotifierWatcher/etc dbus directly.
 // TODO: Make sure the overlay always stays on top. Test with starting the overlay and then opening youtube in fullscreen.
@@ -38,6 +38,100 @@ static void disable_prime_run() {
     unsetenv("__VK_LAYER_NV_optimus");
 }
 
+static std::unique_ptr<gsr::GlobalHotkeysX11> register_x11_hotkeys(gsr::Overlay *overlay) {
+    auto global_hotkeys = std::make_unique<gsr::GlobalHotkeysX11>();
+    const bool show_hotkey_registered = global_hotkeys->bind_key_press({ XK_z, Mod1Mask }, "show_hide", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_show();
+    });
+
+    const bool record_hotkey_registered = global_hotkeys->bind_key_press({ XK_F9, Mod1Mask }, "record", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_record();
+    });
+
+    const bool pause_hotkey_registered = global_hotkeys->bind_key_press({ XK_F7, Mod1Mask }, "pause", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_pause();
+    });
+
+    const bool stream_hotkey_registered = global_hotkeys->bind_key_press({ XK_F8, Mod1Mask }, "stream", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_stream();
+    });
+
+    const bool replay_hotkey_registered = global_hotkeys->bind_key_press({ XK_F10, ShiftMask | Mod1Mask }, "replay_start", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_replay();
+    });
+
+    const bool replay_save_hotkey_registered = global_hotkeys->bind_key_press({ XK_F10, Mod1Mask }, "replay_save", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->save_replay();
+    });
+
+    if(!show_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+z for showing the overlay because the hotkey is registered by another program\n");
+
+    if(!record_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+f9 for recording because the hotkey is registered by another program\n");
+
+    if(!pause_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+f7 for pausing because the hotkey is registered by another program\n");
+
+    if(!stream_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+f8 for streaming because the hotkey is registered by another program\n");
+
+    if(!replay_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+shift+f10 for starting replay because the hotkey is registered by another program\n");
+
+    if(!replay_save_hotkey_registered)
+        fprintf(stderr, "error: failed to register hotkey alt+f10 for saving replay because the hotkey is registered by another program\n");
+
+    if(!show_hotkey_registered || !record_hotkey_registered || !pause_hotkey_registered || !stream_hotkey_registered || !replay_hotkey_registered || !replay_save_hotkey_registered)
+        return nullptr;
+
+    return global_hotkeys;
+}
+
+static std::unique_ptr<gsr::GlobalHotkeysLinux> register_linux_hotkeys(gsr::Overlay *overlay) {
+    auto global_hotkeys = std::make_unique<gsr::GlobalHotkeysLinux>();
+    if(!global_hotkeys->start())
+        fprintf(stderr, "error: failed to start global hotkeys\n");
+
+    global_hotkeys->bind_action("show_hide", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_show();
+    });
+
+    global_hotkeys->bind_action("record", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_record();
+    });
+
+    global_hotkeys->bind_action("pause", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_pause();
+    });
+
+    global_hotkeys->bind_action("stream", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_stream();
+    });
+
+    global_hotkeys->bind_action("replay_start", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->toggle_replay();
+    });
+
+    global_hotkeys->bind_action("replay_save", [overlay](const std::string &id) {
+        fprintf(stderr, "pressed %s\n", id.c_str());
+        overlay->save_replay();
+    });
+
+    return global_hotkeys;
+}
+
 int main(void) {
     setlocale(LC_ALL, "C"); // Sigh... stupid C
 
@@ -89,7 +183,6 @@ int main(void) {
     }
 
     mgl_context *context = mgl_get_context();
-    const int x11_socket = XConnectionNumber((Display*)context->connection);
 
     egl_functions egl_funcs;
     egl_funcs.eglGetError = (decltype(egl_funcs.eglGetError))context->gl.eglGetProcAddress("eglGetError");
@@ -107,96 +200,27 @@ int main(void) {
     auto overlay = std::make_unique<gsr::Overlay>(resources_path, gsr_info, egl_funcs);
     //overlay.show();
 
-    // gsr::GlobalHotkeysX11 global_hotkeys;
-    // const bool show_hotkey_registered = global_hotkeys.bind_key_press({ XK_z, Mod1Mask }, "show_hide", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->toggle_show();
-    // });
+    std::unique_ptr<gsr::GlobalHotkeys> global_hotkeys = nullptr;
+    if(gsr_info.system_info.display_server == gsr::DisplayServer::X11) {
+        global_hotkeys = register_x11_hotkeys(overlay.get());
+        if(!global_hotkeys) {
+            fprintf(stderr, "info: failed to register some x11 hotkeys because they are registered by another program. Will use linux hotkeys instead that can clash with keys used by other applications\n");
+            global_hotkeys = register_linux_hotkeys(overlay.get());
+        }
+    } else {
+        fprintf(stderr, "info: Global linux hotkeys are used which can clash with keys used by other applications. Use X11 instead if this is an issue for you\n");
+        global_hotkeys = register_linux_hotkeys(overlay.get());
+    }
 
-    // const bool record_hotkey_registered = global_hotkeys.bind_key_press({ XK_F9, Mod1Mask }, "record", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->toggle_record();
-    // });
-
-    // const bool pause_hotkey_registered = global_hotkeys.bind_key_press({ XK_F7, Mod1Mask }, "pause", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->toggle_pause();
-    // });
-
-    // const bool stream_hotkey_registered = global_hotkeys.bind_key_press({ XK_F8, Mod1Mask }, "stream", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->toggle_stream();
-    // });
-
-    // const bool replay_hotkey_registered = global_hotkeys.bind_key_press({ XK_F10, ShiftMask | Mod1Mask }, "replay_start", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->toggle_replay();
-    // });
-
-    // const bool replay_save_hotkey_registered = global_hotkeys.bind_key_press({ XK_F10, Mod1Mask }, "replay_save", [&](const std::string &id) {
-    //     fprintf(stderr, "pressed %s\n", id.c_str());
-    //     overlay->save_replay();
-    // });
-
-    gsr::GlobalHotkeysLinux global_hotkeys;
-    if(!global_hotkeys.start())
-        fprintf(stderr, "error: failed to start global hotkeys\n");
-
-    const bool show_hotkey_registered = global_hotkeys.bind_action("show_hide", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->toggle_show();
-    });
-
-    const bool record_hotkey_registered = global_hotkeys.bind_action("record", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->toggle_record();
-    });
-
-    const bool pause_hotkey_registered = global_hotkeys.bind_action("pause", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->toggle_pause();
-    });
-
-    const bool stream_hotkey_registered = global_hotkeys.bind_action("stream", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->toggle_stream();
-    });
-
-    const bool replay_hotkey_registered = global_hotkeys.bind_action("replay_start", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->toggle_replay();
-    });
-
-    const bool replay_save_hotkey_registered = global_hotkeys.bind_action("replay_save", [&](const std::string &id) {
-        fprintf(stderr, "pressed %s\n", id.c_str());
-        overlay->save_replay();
-    });
-
-    if(!show_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+z for showing the overlay because the hotkey is registered by another program\n");
-
-    if(!record_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+f9 for recording because the hotkey is registered by another program\n");
-
-    if(!pause_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+f7 for pausing because the hotkey is registered by another program\n");
-
-    if(!stream_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+f8 for streaming because the hotkey is registered by another program\n");
-
-    if(!replay_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+shift+f10 for starting replay because the hotkey is registered by another program\n");
-
-    if(!replay_save_hotkey_registered)
-        fprintf(stderr, "error: failed to register hotkey alt+f10 for saving replay because the hotkey is registered by another program\n");
+    // TODO: Add hotkeys in Overlay when using x11 global hotkeys. The hotkeys in Overlay should duplicate each key that is used for x11 global hotkeys.
 
     mgl::Clock frame_delta_clock;
     while(running && mgl_is_connected_to_display_server()) {
         const double frame_delta_seconds = frame_delta_clock.restart();
         gsr::set_frame_delta_seconds(frame_delta_seconds);
 
-        global_hotkeys.poll_events();
-        overlay->handle_events();
+        global_hotkeys->poll_events();
+        overlay->handle_events(global_hotkeys.get());
         if(!overlay->draw()) {
             std::this_thread::sleep_for(std::chrono::milliseconds(100));
             mgl_ping_display_server();