App audio capture: remove gsr-app-sink

Connect application/device audio directly to gsr recording node.
This fixes an issue for some users where gsr-app-sink got selected by
default as an output device.

Dont wait until audio node first receives audio before recording audio
from the device. This might fix audio/video desync issue when recording
from microphone for example.

TODO

@@ -294,9 +294,6 @@ Disable GL_DEPTH_TEST, GL_CULL_FACE.
 
 kde plasma portal capture for screenshot doesn't work well because the portal ui is still visible when taking a screenshot because of its animation.
 
-It's possible for microphone audio to get desynced when recording together with desktop audio, when not recording app audio as well.
-    Test recording desktop audio and microphone audio together (-a "default_output|default_input") for around 30 minutes.
-
 We can use dri2connect/dri3open to get the /dev/dri/card device. Note that this doesn't work on nvidia x11.
 
 Add support for QVBR (QP with target bitrate). Maybe use VBR instead, since nvidia doesn't support QVBR and neither does vulkan.
@@ -314,10 +311,6 @@ Set top level window argument for portal capture. Same for gpu-screen-recorder-g
 
 Remove unix domain socket code from kms-client/server and use socketpair directly. To make this possible always execute the kms server permission setup in flatpak, before starting recording (in gpu-screen-recorder-gtk).
 
-Application audio capture isn't good enough. It creates a sink that for some automatically gets selected as the default output device and it's visible as an output device.
-    Fix some of these issues by setting gsr-app-sink media class to "Stream/Input/Audio" and node.virtual=true.
-    However that causes pulseaudio to be unable to record from gsr-app-sink, and it ends up being stuck in pa_sound_device_handle_reconnect in the loop with pa_mainloop_iterate.
-
 Add -k best/best_hdr/best_10bit option, to automatically choose the best codec (prefer av1, then hevc and then h264. For webm files choose vp9 and then vp8).
 
 Check if region capture works properly with fractional scaling on wayland.

include/pipewire_audio.h

@@ -94,18 +94,12 @@ typedef struct {
     size_t num_requested_links;
     size_t requested_links_capacity_items;
 
-    struct pw_proxy **virtual_sink_proxies;
-    size_t num_virtual_sink_proxies;
-    size_t virtual_sink_proxies_capacity_items;
-
     bool running;
 } gsr_pipewire_audio;
 
 bool gsr_pipewire_audio_init(gsr_pipewire_audio *self);
 void gsr_pipewire_audio_deinit(gsr_pipewire_audio *self);
 
-bool gsr_pipewire_audio_create_virtual_sink(gsr_pipewire_audio *self, const char *name);
-
 /*
     This function links audio source outputs from applications that match the name |app_names| to the input
     that matches the name |stream_name_input|.
@@ -140,6 +134,17 @@ bool gsr_pipewire_audio_add_link_from_apps_to_sink(gsr_pipewire_audio *self, con
 */
 bool gsr_pipewire_audio_add_link_from_apps_to_sink_inverted(gsr_pipewire_audio *self, const char **app_names, int num_app_names, const char *sink_name_input);
 
+/*
+    This function links audio source outputs from devices that match the name |source_names| to the input
+    that matches the name |stream_name_input|.
+    If a device or a new device starts outputting audio after this function is called and the device name matches
+    then it will automatically link the audio sources.
+    |source_names| and |stream_name_input| are case-insensitive matches.
+    |source_names| can include "default_output" or "default_input" to use the default output/input
+    and it will automatically switch when the default output/input is changed in system audio settings.
+*/
+bool gsr_pipewire_audio_add_link_from_sources_to_stream(gsr_pipewire_audio *self, const char **source_names, int num_source_names, const char *stream_name_input);
+
 /*
     This function links audio source outputs from devices that match the name |source_names| to the input
     that matches the name |sink_name_input|.

include/sound.hpp

@@ -61,12 +61,12 @@ typedef enum {
 
 /*
     Get a sound device by name, returning the device into the |device| parameter.
-    |device_name| can be a device name or "default_output" or "default_input".
+    |device_name| can be a device name or "default_output", "default_input" or "" to not connect to any device (used for app audio for example).
     If the device name is "default_output" or "default_input" then it will automatically switch which
     device is records from when the default output/input is changed in the system audio settings.
     Returns 0 on success, or a negative value on failure.
 */
-int sound_device_get_by_name(SoundDevice *device, const char *device_name, const char *description, unsigned int num_channels, unsigned int period_frame_size, AudioFormat audio_format);
+int sound_device_get_by_name(SoundDevice *device, const char *node_name, const char *device_name, const char *description, unsigned int num_channels, unsigned int period_frame_size, AudioFormat audio_format);
 
 void sound_device_close(SoundDevice *device);
 

src/main.cpp

@@ -2911,7 +2911,7 @@ static std::vector<AudioDeviceData> create_device_audio_inputs(const std::vector
             audio_device.sound_device.frames = 0;
         } else {
             const std::string description = "gsr-" + audio_input.name;
-            if(sound_device_get_by_name(&audio_device.sound_device, audio_input.name.c_str(), description.c_str(), num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
+            if(sound_device_get_by_name(&audio_device.sound_device, description.c_str(), audio_input.name.c_str(), description.c_str(), num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
                 fprintf(stderr, "gsr error: failed to get \"%s\" audio device\n", audio_input.name.c_str());
                 _exit(1);
             }
@@ -2936,17 +2936,12 @@ static AudioDeviceData create_application_audio_audio_input(const MergedAudioInp
         fprintf(stderr, "gsr error: failed to generate random string\n");
         _exit(1);
     }
+
     std::string combined_sink_name = "gsr-combined-";
     combined_sink_name.append(random_str, sizeof(random_str));
-
-    if(!gsr_pipewire_audio_create_virtual_sink(pipewire_audio, combined_sink_name.c_str())) {
-        fprintf(stderr, "gsr error: failed to create virtual sink for application audio\n");
-        _exit(1);
-    }
-
     combined_sink_name += ".monitor";
 
-    if(sound_device_get_by_name(&audio_device.sound_device, combined_sink_name.c_str(), "gpu-screen-recorder", num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
+    if(sound_device_get_by_name(&audio_device.sound_device, combined_sink_name.c_str(), "", "gpu-screen-recorder", num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
         fprintf(stderr, "gsr error: failed to setup audio recording to combined sink\n");
         _exit(1);
     }
@@ -2967,19 +2962,19 @@ static AudioDeviceData create_application_audio_audio_input(const MergedAudioInp
     }
 
     if(!audio_devices_sources.empty()) {
-        if(!gsr_pipewire_audio_add_link_from_sources_to_sink(pipewire_audio, audio_devices_sources.data(), audio_devices_sources.size(), combined_sink_name.c_str())) {
+        if(!gsr_pipewire_audio_add_link_from_sources_to_stream(pipewire_audio, audio_devices_sources.data(), audio_devices_sources.size(), combined_sink_name.c_str())) {
             fprintf(stderr, "gsr error: failed to add application audio link\n");
             _exit(1);
         }
     }
 
     if(app_audio_inverted) {
-        if(!gsr_pipewire_audio_add_link_from_apps_to_sink_inverted(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
+        if(!gsr_pipewire_audio_add_link_from_apps_to_stream_inverted(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
             fprintf(stderr, "gsr error: failed to add application audio link\n");
             _exit(1);
         }
     } else {
-        if(!gsr_pipewire_audio_add_link_from_apps_to_sink(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
+        if(!gsr_pipewire_audio_add_link_from_apps_to_stream(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
             fprintf(stderr, "gsr error: failed to add application audio link\n");
             _exit(1);
         }

src/pipewire_audio.c

@@ -58,6 +58,29 @@ static bool requested_link_matches_name_case_insensitive(const gsr_pipewire_audi
     return false;
 }
 
+static bool requested_link_matches_name_case_insensitive_any_type(const gsr_pipewire_audio *self, const gsr_pipewire_audio_requested_link *requested_link, const char *name) {
+    for(int i = 0; i < requested_link->num_outputs; ++i) {
+        switch(requested_link->outputs[i].type) {
+            case GSR_PIPEWIRE_AUDIO_REQUESTED_TYPE_STANDARD: {
+                if(strcasecmp(requested_link->outputs[i].name, name) == 0)
+                    return true;
+                break;
+            }
+            case GSR_PIPEWIRE_AUDIO_REQUESTED_TYPE_DEFAULT_OUTPUT: {
+                if(strcasecmp(self->default_output_device_name, name) == 0)
+                    return true;
+                break;
+            }
+            case GSR_PIPEWIRE_AUDIO_REQUESTED_TYPE_DEFAULT_INPUT: {
+                if(strcasecmp(self->default_input_device_name, name) == 0)
+                    return true;
+                break;
+            }
+        }
+    }
+    return false;
+}
+
 static bool requested_link_has_type(const gsr_pipewire_audio_requested_link *requested_link, gsr_pipewire_audio_requested_type type) {
     for(int i = 0; i < requested_link->num_outputs; ++i) {
         if(requested_link->outputs[i].type == type)
@@ -168,7 +191,7 @@ static void gsr_pipewire_audio_create_link(gsr_pipewire_audio *self, const gsr_p
         if(output_node->type != requested_link->output_type)
             continue;
 
-        const bool requested_link_matches_app = requested_link_matches_name_case_insensitive(requested_link, output_node->name);
+        const bool requested_link_matches_app = requested_link_matches_name_case_insensitive_any_type(self, requested_link, output_node->name);
         if(requested_link->inverted) {
             if(requested_link_matches_app)
                 continue;
@@ -642,20 +665,6 @@ void gsr_pipewire_audio_deinit(gsr_pipewire_audio *self) {
         pw_thread_loop_stop(self->thread_loop);
     }
 
-    for(size_t i = 0; i < self->num_virtual_sink_proxies; ++i) {
-        if(self->virtual_sink_proxies[i]) {
-            pw_proxy_destroy(self->virtual_sink_proxies[i]);
-            self->virtual_sink_proxies[i] = NULL;
-        }
-    }
-    self->num_virtual_sink_proxies = 0;
-    self->virtual_sink_proxies_capacity_items = 0;
-
-    if(self->virtual_sink_proxies) {
-        free(self->virtual_sink_proxies);
-        self->virtual_sink_proxies = NULL;
-    }
-
     if(self->metadata_proxy) {
         spa_hook_remove(&self->metadata_listener);
         spa_hook_remove(&self->metadata_proxy_listener);
@@ -733,54 +742,6 @@ void gsr_pipewire_audio_deinit(gsr_pipewire_audio *self) {
 #endif
 }
 
-static struct pw_properties* gsr_pipewire_create_null_audio_sink(const char *name) {
-    char props_str[512];
-    snprintf(props_str, sizeof(props_str),
-        "{ factory.name=support.null-audio-sink node.name=\"%s\" media.class=Audio/Sink object.linger=false audio.position=[FL FR]"
-        " monitor.channel-volumes=true monitor.passthrough=true adjust_time=0 node.description=gsr-app-sink slaves=\"\" priority.driver=1 priority.session=1 }", name);
-    struct pw_properties *props = pw_properties_new_string(props_str);
-    if(!props) {
-        fprintf(stderr, "gsr error: gsr_pipewire_create_null_audio_sink: failed to create virtual sink properties\n");
-        return NULL;
-    }
-    return props;
-}
-
-bool gsr_pipewire_audio_create_virtual_sink(gsr_pipewire_audio *self, const char *name) {
-    if(!array_ensure_capacity((void**)&self->virtual_sink_proxies, self->num_virtual_sink_proxies, &self->virtual_sink_proxies_capacity_items, sizeof(struct pw_proxy*)))
-        return false;
-
-    pw_thread_loop_lock(self->thread_loop);
-
-    struct pw_properties *virtual_sink_props = gsr_pipewire_create_null_audio_sink(name);
-    if(!virtual_sink_props) {
-        pw_thread_loop_unlock(self->thread_loop);
-        return false;
-    }
-
-    struct pw_proxy *virtual_sink_proxy = pw_core_create_object(self->core, "adapter", PW_TYPE_INTERFACE_Node, PW_VERSION_NODE, &virtual_sink_props->dict, 0);
-    // TODO:
-    // If these are done then the above needs sizeof(*self) as the last argument
-    //pw_proxy_add_object_listener(virtual_sink_proxy, &pd->object_listener, &node_events, self);
-	//pw_proxy_add_listener(virtual_sink_proxy, &pd->proxy_listener, &proxy_events, self);
-    // TODO: proxy
-    pw_properties_free(virtual_sink_props);
-    if(!virtual_sink_proxy) {
-        fprintf(stderr, "gsr error: gsr_pipewire_audio_create_virtual_sink: failed to create virtual sink\n");
-        pw_thread_loop_unlock(self->thread_loop);
-        return false;
-    }
-
-    self->server_version_sync = pw_core_sync(self->core, PW_ID_CORE, self->server_version_sync);
-    pw_thread_loop_wait(self->thread_loop);
-    pw_thread_loop_unlock(self->thread_loop);
-
-    self->virtual_sink_proxies[self->num_virtual_sink_proxies] = virtual_sink_proxy;
-    ++self->num_virtual_sink_proxies;
-
-    return true;
-}
-
 static bool string_remove_suffix(char *str, const char *suffix) {
     int str_len = strlen(str);
     int suffix_len = strlen(suffix);
@@ -834,6 +795,7 @@ static bool gsr_pipewire_audio_add_links_to_output(gsr_pipewire_audio *self, con
     self->requested_links[self->num_requested_links].inverted = inverted;
     ++self->num_requested_links;
     gsr_pipewire_audio_create_link(self, &self->requested_links[self->num_requested_links - 1]);
+    // TODO: Remove these?
     gsr_pipewire_audio_create_link_for_default_devices(self, &self->requested_links[self->num_requested_links - 1], GSR_PIPEWIRE_AUDIO_REQUESTED_TYPE_DEFAULT_OUTPUT);
     gsr_pipewire_audio_create_link_for_default_devices(self, &self->requested_links[self->num_requested_links - 1], GSR_PIPEWIRE_AUDIO_REQUESTED_TYPE_DEFAULT_INPUT);
     pw_thread_loop_unlock(self->thread_loop);
@@ -865,6 +827,10 @@ bool gsr_pipewire_audio_add_link_from_apps_to_sink_inverted(gsr_pipewire_audio *
     return gsr_pipewire_audio_add_links_to_output(self, app_names, num_app_names, sink_name_input, GSR_PIPEWIRE_AUDIO_NODE_TYPE_STREAM_OUTPUT, GSR_PIPEWIRE_AUDIO_LINK_INPUT_TYPE_SINK, true);
 }
 
+bool gsr_pipewire_audio_add_link_from_sources_to_stream(gsr_pipewire_audio *self, const char **source_names, int num_source_names, const char *stream_name_input) {
+    return gsr_pipewire_audio_add_links_to_output(self, source_names, num_source_names, stream_name_input, GSR_PIPEWIRE_AUDIO_NODE_TYPE_SINK_OR_SOURCE, GSR_PIPEWIRE_AUDIO_LINK_INPUT_TYPE_STREAM, false);
+}
+
 bool gsr_pipewire_audio_add_link_from_sources_to_sink(gsr_pipewire_audio *self, const char **source_names, int num_source_names, const char *sink_name_input) {
     return gsr_pipewire_audio_add_links_to_output(self, source_names, num_source_names, sink_name_input, GSR_PIPEWIRE_AUDIO_NODE_TYPE_SINK_OR_SOURCE, GSR_PIPEWIRE_AUDIO_LINK_INPUT_TYPE_SINK, false);
 }

src/sound.cpp

@@ -189,7 +189,7 @@ static pa_handle* pa_sound_device_new(const char *server,
     snprintf(p->stream_name, sizeof(p->stream_name), "%s", stream_name);
 
     p->reconnect = true;
-    p->reconnect_last_tried_seconds = clock_get_monotonic_seconds() - 1000.0;
+    p->reconnect_last_tried_seconds = clock_get_monotonic_seconds() - (RECONNECT_TRY_TIMEOUT_SECONDS * 1000.0 * 2.0);
     p->default_output_device_name[0] = '\0';
     p->default_input_device_name[0] = '\0';
     p->device_type = DeviceType::STANDARD;
@@ -208,6 +208,10 @@ static pa_handle* pa_sound_device_new(const char *server,
 
     pa_proplist *proplist = pa_proplist_new();
     pa_proplist_sets(proplist, PA_PROP_MEDIA_ROLE, "production");
+    if(strcmp(device_name, "") == 0) {
+        pa_proplist_sets(proplist, "node.autoconnect", "false");
+        pa_proplist_sets(proplist, "node.dont-reconnect", "true");
+    }
 
     if (!(p->mainloop = pa_mainloop_new()))
         goto fail;
@@ -288,20 +292,6 @@ static bool pa_sound_device_handle_reconnect(pa_handle *p, char *device_name, si
         return false;
     }
 
-    for(;;) {
-        pa_stream_state_t state = pa_stream_get_state(p->stream);
-
-        if(state == PA_STREAM_READY)
-            break;
-
-        if(!PA_STREAM_IS_GOOD(state)) {
-            //pa_context_errno(p->context);
-            return false;
-        }
-
-        pa_mainloop_iterate(p->mainloop, 1, NULL);
-    }
-
     std::lock_guard<std::mutex> lock(p->reconnect_mutex);
     p->reconnect = false;
     return true;
@@ -322,6 +312,11 @@ static int pa_sound_device_read(pa_handle *p, double timeout_seconds) {
     if(!pa_sound_device_handle_reconnect(p, device_name, sizeof(device_name), start_time))
         goto fail;
 
+    if(pa_stream_get_state(p->stream) != PA_STREAM_READY) {
+        pa_mainloop_iterate(p->mainloop, 0, NULL);
+        goto fail;
+    }
+
     CHECK_DEAD_GOTO(p, rerror, fail);
 
     while (p->output_index < p->output_length) {
@@ -415,7 +410,7 @@ static int audio_format_to_get_bytes_per_sample(AudioFormat audio_format) {
     return 2;
 }
 
-int sound_device_get_by_name(SoundDevice *device, const char *device_name, const char *description, unsigned int num_channels, unsigned int period_frame_size, AudioFormat audio_format) {
+int sound_device_get_by_name(SoundDevice *device, const char *node_name, const char *device_name, const char *description, unsigned int num_channels, unsigned int period_frame_size, AudioFormat audio_format) {
     pa_sample_spec ss;
     ss.format = audio_format_to_pulse_audio_format(audio_format);
     ss.rate = 48000;
@@ -429,7 +424,7 @@ int sound_device_get_by_name(SoundDevice *device, const char *device_name, const
     buffer_attr.maxlength = buffer_attr.fragsize;
 
     int error = 0;
-    pa_handle *handle = pa_sound_device_new(nullptr, description, device_name, description, &ss, &buffer_attr, &error);
+    pa_handle *handle = pa_sound_device_new(nullptr, node_name, device_name, description, &ss, &buffer_attr, &error);
     if(!handle) {
         fprintf(stderr, "gsr error: pa_sound_device_new() failed: %s. Audio input device %s might not be valid\n", pa_strerror(error), device_name);
         return -1;