Return status code from dbus instead of true/false

include/dbus.h

@@ -35,9 +35,10 @@ bool gsr_dbus_init(gsr_dbus *self, const char *screencast_restore_token);
 void gsr_dbus_deinit(gsr_dbus *self);
 
 /* The follow functions should be called in order to setup ScreenCast properly */
-bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle);
-bool gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handle, gsr_portal_capture_type capture_type, gsr_portal_cursor_mode cursor_mode);
-bool gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint32_t *pipewire_node);
+/* These functions that return an int return the response status code */
+int gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle);
+int gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handle, gsr_portal_capture_type capture_type, gsr_portal_cursor_mode cursor_mode);
+int gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint32_t *pipewire_node);
 bool gsr_dbus_screencast_open_pipewire_remote(gsr_dbus *self, const char *session_handle, int *pipewire_fd);
 const char* gsr_dbus_screencast_get_restore_token(gsr_dbus *self);
 

src/capture/portal.c

@@ -143,8 +143,6 @@ static void gsr_capture_portal_get_restore_token_from_cache(char *buffer, size_t
     long file_size = ftell(f);
     fseek(f, 0, SEEK_SET);
 
-    fprintf(stderr, "file size: %ld\n", file_size);
-
     if(file_size > 0 && file_size < 1024 && file_size < (long)buffer_size && (long)fread(buffer, 1, file_size, f) != file_size) {
         buffer[0] = '\0';
         fprintf(stderr, "gsr warning: gsr_capture_portal_get_restore_token_from_cache: failed to read restore token (%s)\n", restore_token_path);
@@ -159,9 +157,10 @@ static void gsr_capture_portal_get_restore_token_from_cache(char *buffer, size_t
     fclose(f);
 }
 
-static bool gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewire_fd, uint32_t *pipewire_node) {
+static int gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewire_fd, uint32_t *pipewire_node) {
     *pipewire_fd = 0;
     *pipewire_node = 0;
+    int response_status = 0;
 
     char restore_token[1024];
     restore_token[0] = '\0';
@@ -169,24 +168,27 @@ static bool gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewir
         gsr_capture_portal_get_restore_token_from_cache(restore_token, sizeof(restore_token));
 
     if(!gsr_dbus_init(&self->dbus, restore_token))
-        return false;
+        return -1;
 
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: CreateSession\n");
-    if(!gsr_dbus_screencast_create_session(&self->dbus, &self->session_handle)) {
+    response_status = gsr_dbus_screencast_create_session(&self->dbus, &self->session_handle);
+    if(response_status != 0) {
         fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: CreateSession failed\n");
-        return false;
+        return response_status;
     }
 
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: SelectSources\n");
-    if(!gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_EMBEDDED : GSR_PORTAL_CURSOR_MODE_HIDDEN)) {
+    response_status = gsr_dbus_screencast_select_sources(&self->dbus, self->session_handle, GSR_PORTAL_CAPTURE_TYPE_ALL, self->params.record_cursor ? GSR_PORTAL_CURSOR_MODE_EMBEDDED : GSR_PORTAL_CURSOR_MODE_HIDDEN);
+    if(response_status != 0) {
         fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: SelectSources failed\n");
-        return false;
+        return response_status;
     }
 
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: Start\n");
-    if(!gsr_dbus_screencast_start(&self->dbus, self->session_handle, pipewire_node)) {
+    response_status = gsr_dbus_screencast_start(&self->dbus, self->session_handle, pipewire_node);
+    if(response_status != 0) {
         fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: Start failed\n");
-        return false;
+        return response_status;
     }
 
     const char *screencast_restore_token = gsr_dbus_screencast_get_restore_token(&self->dbus);
@@ -196,11 +198,11 @@ static bool gsr_capture_portal_setup_dbus(gsr_capture_portal *self, int *pipewir
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: OpenPipeWireRemote\n");
     if(!gsr_dbus_screencast_open_pipewire_remote(&self->dbus, self->session_handle, pipewire_fd)) {
         fprintf(stderr, "gsr error: gsr_capture_portal_setup_dbus: OpenPipeWireRemote failed\n");
-        return false;
+        return -1;
     }
 
     fprintf(stderr, "gsr info: gsr_capture_portal_setup_dbus: desktop portal setup finished\n");
-    return true;
+    return 0;
 }
 
 static bool gsr_capture_portal_get_frame_dimensions(gsr_capture_portal *self) {
@@ -236,10 +238,20 @@ static int gsr_capture_portal_start(gsr_capture *cap, AVCodecContext *video_code
 
     int pipewire_fd = 0;
     uint32_t pipewire_node = 0;
-    if(!gsr_capture_portal_setup_dbus(self, &pipewire_fd, &pipewire_node)) {
-        fprintf(stderr, "gsr error: gsr_capture_portal_start: desktop portal capture failed. Either you canceled the desktop portal or your Wayland compositor doesn't support desktop portal capture or it's incorrectly setup on your system\n");
+    const int response_status = gsr_capture_portal_setup_dbus(self, &pipewire_fd, &pipewire_node);
+    if(response_status != 0) {
         gsr_capture_portal_stop(self);
-        return 50;
+        // Response status values:
+        // 0: Success, the request is carried out
+        // 1: The user cancelled the interaction
+        // 2: The user interaction was ended in some other way
+        // Response status value 2 happens usually if there was some kind of error in the desktop portal on the system
+        if(response_status == 2) {
+            fprintf(stderr, "gsr error: gsr_capture_portal_start: desktop portal capture failed. Either you canceled the desktop portal or your Wayland compositor doesn't support desktop portal capture or it's incorrectly setup on your system\n");
+            return 50;
+        } else {
+            return -1;
+        }
     }
 
     fprintf(stderr, "gsr info: gsr_capture_portal_start: setting up pipewire\n");

src/dbus.c

@@ -413,21 +413,17 @@ static bool gsr_dbus_call_screencast_method(gsr_dbus *self, const char *method_n
     return true;
 }
 
-static bool gsr_dbus_response_status_ok(DBusMessageIter *resp_args) {
+static int gsr_dbus_get_response_status(DBusMessageIter *resp_args) {
     if(dbus_message_iter_get_arg_type(resp_args) != DBUS_TYPE_UINT32) {
-        fprintf(stderr, "gsr error: gsr_dbus_extract_desktop_portal_response_is_ok: missing uint32 in response\n");
-        return false;
+        fprintf(stderr, "gsr error: gsr_dbus_get_response_status: missing uint32 in response\n");
+        return -1;
     }
 
     dbus_uint32_t response_status = 0;
     dbus_message_iter_get_basic(resp_args, &response_status);
-    if(response_status != 0) {
-        fprintf(stderr, "gsr error: gsr_dbus_extract_desktop_portal_response_is_ok: got status: %d, expected 0\n", response_status);
-        return false;
-    }
 
     dbus_message_iter_next(resp_args);
-    return true;
+    return (int)response_status;
 }
 
 static dict_entry* find_dict_entry_by_key(dict_entry *entries, int num_entries, const char *key) {
@@ -569,7 +565,7 @@ static bool gsr_dbus_get_map(DBusMessageIter *resp_args, dict_entry *entries, in
     return false;
 }
 
-bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
+int gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
     assert(session_handle);
     *session_handle = NULL;
 
@@ -591,7 +587,7 @@ bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
     DBusMessage *response_msg = NULL;
     if(!gsr_dbus_call_screencast_method(self, "CreateSession", NULL, NULL, args, 2, NULL, &response_msg)) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_create_session: failed to setup ScreenCast session. Make sure you have a desktop portal running with support for the ScreenCast interface and that the desktop portal matches the Wayland compositor you are running.\n");
-        return false;
+        return -1;
     }
 
     // TODO: Verify signal path matches |res|, maybe check the below
@@ -601,13 +597,13 @@ bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
     if(!dbus_message_iter_init(response_msg, &resp_args)) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_create_session: missing response\n");
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
-    if(!gsr_dbus_response_status_ok(&resp_args)) {
-        fprintf(stderr, "gsr error: gsr_dbus_screencast_create_session: failed to setup ScreenCast session. Make sure you have a desktop portal running with support for the ScreenCast interface and that the desktop portal matches the Wayland compositor you are running.\n");
+    const int response_status = gsr_dbus_get_response_status(&resp_args);
+    if(response_status != 0) {
         dbus_message_unref(response_msg);
-        return false;
+        return response_status;
     }
 
     dict_entry entries[1];
@@ -616,13 +612,13 @@ bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
     entries[0].value_type = DICT_TYPE_STRING;
     if(!gsr_dbus_get_map(&resp_args, entries, 1)) {
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
     if(!entries[0].str) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_create_session: missing \"session_handle\" in response\n");
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
     *session_handle = entries[0].str;
@@ -630,10 +626,10 @@ bool gsr_dbus_screencast_create_session(gsr_dbus *self, char **session_handle) {
     //free(entries[0].str);
 
     dbus_message_unref(response_msg);
-    return true;
+    return 0;
 }
 
-bool gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handle, gsr_portal_capture_type capture_type, gsr_portal_cursor_mode cursor_mode) {
+int gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_handle, gsr_portal_capture_type capture_type, gsr_portal_cursor_mode cursor_mode) {
     assert(session_handle);
 
     char handle_token[64];
@@ -682,9 +678,9 @@ bool gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_hand
             fprintf(stderr, "gsr warning: gsr_dbus_screencast_select_sources: SelectSources failed, retrying without restore_token\n");
             num_arg_dict = 5;
             if(!gsr_dbus_call_screencast_method(self, "SelectSources", session_handle, NULL, args, num_arg_dict, NULL, &response_msg))
-                return false;
+                return -1;
         } else {
-            return false;
+            return -1;
         }
     }
 
@@ -694,16 +690,18 @@ bool gsr_dbus_screencast_select_sources(gsr_dbus *self, const char *session_hand
     if(!dbus_message_iter_init(response_msg, &resp_args)) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_create_session: missing response\n");
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
-    if(!gsr_dbus_response_status_ok(&resp_args)) {
+    
+    const int response_status = gsr_dbus_get_response_status(&resp_args);
+    if(response_status != 0) {
         dbus_message_unref(response_msg);
-        return false;
+        return response_status;
     }
 
     dbus_message_unref(response_msg);
-    return true;
+    return 0;
 }
 
 static dbus_uint32_t screencast_stream_get_pipewire_node(DBusMessageIter *iter) {
@@ -724,7 +722,7 @@ static dbus_uint32_t screencast_stream_get_pipewire_node(DBusMessageIter *iter)
     return 0;
 }
 
-bool gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint32_t *pipewire_node) {
+int gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint32_t *pipewire_node) {
     assert(session_handle);
     *pipewire_node = 0;
 
@@ -738,7 +736,7 @@ bool gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint3
     
     DBusMessage *response_msg = NULL;
     if(!gsr_dbus_call_screencast_method(self, "Start", session_handle, "", args, 1, NULL, &response_msg))
-        return false;
+        return -1;
 
     // TODO: Verify signal path matches |res|, maybe check the below
     //fprintf(stderr, "signature: %s, sender: %s\n", dbus_message_get_signature(msg), dbus_message_get_sender(msg));
@@ -746,18 +744,19 @@ bool gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint3
     if(!dbus_message_iter_init(response_msg, &resp_args)) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_start: missing response\n");
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
-    if(!gsr_dbus_response_status_ok(&resp_args)) {
+    const int response_status = gsr_dbus_get_response_status(&resp_args);
+    if(response_status != 0) {
         dbus_message_unref(response_msg);
-        return false;
+        return response_status;
     }
 
     if(dbus_message_iter_get_arg_type(&resp_args) != DBUS_TYPE_ARRAY) {
         fprintf(stderr, "gsr error: gsr_dbus_screencast_start: missing array in response\n");
         dbus_message_unref(response_msg);
-        return false;
+        return -1;
     }
 
     DBusMessageIter subiter;
@@ -859,11 +858,11 @@ bool gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint3
     }
 
     dbus_message_unref(response_msg);
-    return true;
+    return 0;
 
     error:
     dbus_message_unref(response_msg);
-    return false;
+    return -1;
 }
 
 bool gsr_dbus_screencast_open_pipewire_remote(gsr_dbus *self, const char *session_handle, int *pipewire_fd) {