Refactor kms_vaapi and kms_cuda

Also fixes color metadata, color range and hdr on nvidia wayland.

include/capture/capture.h

@@ -1,17 +1,24 @@
 #ifndef GSR_CAPTURE_CAPTURE_H
 #define GSR_CAPTURE_CAPTURE_H
 
+#include "../color_conversion.h"
 #include <stdbool.h>
 
 typedef struct AVCodecContext AVCodecContext;
 typedef struct AVFrame AVFrame;
+typedef void* VADisplay;
+typedef struct _VADRMPRIMESurfaceDescriptor VADRMPRIMESurfaceDescriptor;
+typedef struct gsr_cuda gsr_cuda;
+typedef struct AVFrame AVFrame;
+typedef struct CUgraphicsResource_st *CUgraphicsResource;
+typedef struct CUarray_st *CUarray;
 
 typedef struct gsr_capture gsr_capture;
 
 struct gsr_capture {
     /* These methods should not be called manually. Call gsr_capture_* instead */
-    int (*start)(gsr_capture *cap, AVCodecContext *video_codec_context);
-    void (*tick)(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame); /* can be NULL */
+    int (*start)(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame *frame);
+    void (*tick)(gsr_capture *cap, AVCodecContext *video_codec_context); /* can be NULL */
     bool (*should_stop)(gsr_capture *cap, bool *err); /* can be NULL */
     int (*capture)(gsr_capture *cap, AVFrame *frame);
     void (*capture_end)(gsr_capture *cap, AVFrame *frame); /* can be NULL */
@@ -21,12 +28,34 @@ struct gsr_capture {
     bool started;
 };
 
-int gsr_capture_start(gsr_capture *cap, AVCodecContext *video_codec_context);
-void gsr_capture_tick(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame);
+typedef struct gsr_capture_base gsr_capture_base;
+
+struct gsr_capture_base {
+    unsigned int input_texture;
+    unsigned int target_textures[2];
+    unsigned int cursor_texture;
+
+    gsr_color_conversion color_conversion;
+
+    AVCodecContext *video_codec_context;
+};
+
+typedef struct {
+    gsr_cuda *cuda;
+    CUgraphicsResource *cuda_graphics_resources;
+    CUarray *mapped_arrays;
+} gsr_cuda_context;
+
+int gsr_capture_start(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame *frame);
+void gsr_capture_tick(gsr_capture *cap, AVCodecContext *video_codec_context);
 bool gsr_capture_should_stop(gsr_capture *cap, bool *err);
 int gsr_capture_capture(gsr_capture *cap, AVFrame *frame);
 void gsr_capture_end(gsr_capture *cap, AVFrame *frame);
 /* Calls |gsr_capture_stop| as well */
 void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context);
 
+bool gsr_capture_base_setup_vaapi_textures(gsr_capture_base *self, AVFrame *frame, gsr_egl *egl, VADisplay va_dpy, VADRMPRIMESurfaceDescriptor *prime, gsr_color_range color_range);
+bool gsr_capture_base_setup_cuda_textures(gsr_capture_base *base, AVFrame *frame, gsr_cuda_context *cuda_context, gsr_egl *egl, gsr_color_range color_range, bool hdr);
+void gsr_capture_base_stop(gsr_capture_base *self, gsr_egl *egl);
+
 #endif /* GSR_CAPTURE_CAPTURE_H */

include/capture/kms.h

@@ -0,0 +1,48 @@
+#ifndef GSR_CAPTURE_KMS_H
+#define GSR_CAPTURE_KMS_H
+
+#include "../../kms/client/kms_client.h"
+#include "../color_conversion.h"
+#include "../vec2.h"
+#include "../defs.h"
+#include <stdbool.h>
+
+typedef struct gsr_capture_base gsr_capture_base;
+typedef struct AVCodecContext AVCodecContext;
+typedef struct AVMasteringDisplayMetadata AVMasteringDisplayMetadata;
+typedef struct AVContentLightMetadata AVContentLightMetadata;
+typedef struct gsr_capture_kms gsr_capture_kms;
+typedef struct gsr_egl gsr_egl;
+typedef struct AVFrame AVFrame;
+
+#define MAX_CONNECTOR_IDS 32
+
+typedef struct {
+    uint32_t connector_ids[MAX_CONNECTOR_IDS];
+    int num_connector_ids;
+} MonitorId;
+
+struct gsr_capture_kms {
+    bool should_stop;
+    bool stop_is_error;
+    
+    gsr_kms_client kms_client;
+    gsr_kms_response kms_response;
+
+    vec2i capture_pos;
+    vec2i capture_size;
+    MonitorId monitor_id;
+
+    AVMasteringDisplayMetadata *mastering_display_metadata;
+    AVContentLightMetadata *light_metadata;
+
+    gsr_monitor_rotation monitor_rotation;
+};
+
+/* Returns 0 on success */
+int gsr_capture_kms_start(gsr_capture_kms *self, gsr_capture_base *base, const char *display_to_capture, gsr_egl *egl, AVCodecContext *video_codec_context);
+void gsr_capture_kms_stop(gsr_capture_kms *self);
+bool gsr_capture_kms_capture(gsr_capture_kms *self, gsr_capture_base *base, AVFrame *frame, gsr_egl *egl, bool hdr, bool screen_plane_use_modifiers, bool cursor_texture_is_external);
+void gsr_capture_kms_cleanup_kms_fds(gsr_capture_kms *self);
+
+#endif /* GSR_CAPTURE_KMS_H */

include/capture/kms_cuda.h

@@ -3,6 +3,7 @@
 
 #include "../vec2.h"
 #include "../utils.h"
+#include "../color_conversion.h"
 #include "capture.h"
 
 typedef struct _XDisplay Display;
@@ -12,6 +13,7 @@ typedef struct {
     const char *display_to_capture; /* if this is "screen", then the first monitor is captured. A copy is made of this */
     gsr_gpu_info gpu_inf;
     bool hdr;
+    gsr_color_range color_range;
 } gsr_capture_kms_cuda_params;
 
 gsr_capture* gsr_capture_kms_cuda_create(const gsr_capture_kms_cuda_params *params);

include/capture/kms_vaapi.h

@@ -12,7 +12,6 @@ typedef struct {
     gsr_egl *egl;
     const char *display_to_capture; /* if this is "screen", then the first monitor is captured. A copy is made of this */
     gsr_gpu_info gpu_inf;
-    bool wayland;
     bool hdr;
     gsr_color_range color_range;
 } gsr_capture_kms_vaapi_params;