Fix -fallback-cpu-encoding not working properly

TODO

@@ -195,8 +195,6 @@ Always disable prime run/dri prime and list all monitors to record from from all
 Allow flv av1 if recent ffmpeg version and streaming to youtube (and twitch?) and for custom services.
 Use explicit sync in pipewire video code: https://docs.pipewire.org/page_dma_buf.html.
 
-Support vaapi rotation. Support for it is added in mesa here: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32919.
-
 Replay (and recording?) fails to save properly sometimes (especially for long videos). This is noticable with mp4 files since they get corrupt and become unplayable.
     The entire video does seem to get saved (it's a large video file) and it seems to have the correct headers but it's not playable.
 
@@ -354,3 +352,7 @@ In gpu screen recorder --info output codec max resolutions. This allows for bett
 Set minimum fps for live stream or piping or always.
 
 Support youtube sso.
+
+Remove -fm content (support it but remove it from documentation and output deprecation notice when its used) and use it when using -fm vbr (which is the default option).
+    But first -fm content needs to be support on wayland as well, by checking if there is a difference between frames (checksum the frame content).
+    -fm content also needs to have a minimum fps to prevent live stream from timing out when nothing changes on the screen.

meson.build

@@ -1,4 +1,4 @@
-project('gpu-screen-recorder', ['c', 'cpp'], version : '5.9.1', default_options : ['warning_level=2'])
+project('gpu-screen-recorder', ['c', 'cpp'], version : '5.9.2', default_options : ['warning_level=2'])
 
 add_project_arguments('-Wshadow', language : ['c', 'cpp'])
 if get_option('buildtype') == 'debug'

project.conf

@@ -1,7 +1,7 @@
 [package]
 name = "gpu-screen-recorder"
 type = "executable"
-version = "5.9.1"
+version = "5.9.2"
 platforms = ["posix"]
 
 [config]

src/main.cpp

@@ -1645,9 +1645,22 @@ static bool get_supported_video_codecs(gsr_egl *egl, gsr_video_codec video_codec
     return false;
 }
 
+static void force_cpu_encoding(args_parser *args_parser) {
+    args_parser->video_codec = GSR_VIDEO_CODEC_H264;
+    args_parser->video_encoder = GSR_VIDEO_ENCODER_HW_CPU;
+    if(args_parser->bitrate_mode == GSR_BITRATE_MODE_VBR) {
+        fprintf(stderr, "gsr warning: bitrate mode has been forcefully set to qp because software encoding option doesn't support vbr option\n");
+        args_parser->bitrate_mode = GSR_BITRATE_MODE_QP;
+    }
+}
+
 static bool get_supported_video_codecs_with_cpu_fallback(gsr_egl *egl, args_parser *args_parser, bool cleanup, gsr_supported_video_codecs *video_codecs) {
-    if(get_supported_video_codecs(egl, args_parser->video_codec, args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU, cleanup, video_codecs))
+    if(get_supported_video_codecs(egl, args_parser->video_codec, args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU, cleanup, video_codecs)) {
+        if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU || !args_parser->fallback_cpu_encoding)
             return true;
+        else if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_GPU && video_codecs->h264.supported && (args_parser->video_codec == (gsr_video_codec)GSR_VIDEO_CODEC_AUTO || args_parser->video_codec == GSR_VIDEO_CODEC_H264))
+            return true;
+    }
 
     if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU || !args_parser->fallback_cpu_encoding)
         return false;
@@ -1660,11 +1673,7 @@ static bool get_supported_video_codecs_with_cpu_fallback(gsr_egl *egl, args_pars
             args_parser->video_codec = GSR_VIDEO_CODEC_H264;
         }
 
-        args_parser->video_encoder = GSR_VIDEO_ENCODER_HW_CPU;
+        force_cpu_encoding(args_parser);
-        if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU && args_parser->bitrate_mode == GSR_BITRATE_MODE_VBR) {
-            fprintf(stderr, "gsr warning: bitrate mode has been forcefully set to qp because software encoding option doesn't support vbr option\n");
-            args_parser->bitrate_mode = GSR_BITRATE_MODE_QP;
-        }
         return true;
     }
 
@@ -2757,9 +2766,7 @@ static void print_codec_error(gsr_video_codec video_codec) {
         "  You can test this by running 'vainfo | grep VAEntrypointEncSlice' to see if it matches any H264/HEVC/AV1/VP8/VP9 profile.\n"
         "  On such distros, you need to manually install mesa from source to enable H264/HEVC hardware acceleration, or use a more user friendly distro. Alternatively record with AV1 if supported by your GPU.\n"
         "  You can alternatively use the flatpak version of GPU Screen Recorder (https://flathub.org/apps/com.dec05eba.gpu_screen_recorder) which bypasses system issues with patented H264/HEVC codecs.\n"
-        "  Make sure you have mesa-extra freedesktop runtime installed when using the flatpak (this should be the default), which can be installed with this command:\n"
+        "  If your GPU doesn't support hardware accelerated video encoding then you can use '-fallback-cpu-encoding yes' option to encode with your cpu instead.\n", video_codec_name, video_codec_name, video_codec_name);
-        "  flatpak install --system org.freedesktop.Platform.GL.default//23.08-extra\n"
-        "  If your GPU doesn't support hardware accelerated video encoding then you can use '-encoder cpu' option to encode with your cpu instead.\n", video_codec_name, video_codec_name, video_codec_name);
 }
 
 static const AVCodec* pick_video_codec(gsr_egl *egl, args_parser *args_parser, bool use_fallback_codec, bool *low_power, gsr_supported_video_codecs *supported_video_codecs) {
@@ -2770,8 +2777,11 @@ static const AVCodec* pick_video_codec(gsr_egl *egl, args_parser *args_parser, b
     if(!video_codec_f && use_fallback_codec && args_parser->video_encoder != GSR_VIDEO_ENCODER_HW_CPU) {
         switch(args_parser->video_codec) {
             case GSR_VIDEO_CODEC_H264: {
-                fprintf(stderr, "gsr warning: selected video codec h264 is not supported, trying hevc instead\n");
+                fprintf(stderr, "gsr error: selected video codec h264 is not supported\n");
-                args_parser->video_codec = GSR_VIDEO_CODEC_HEVC;
+                if(args_parser->fallback_cpu_encoding) {
+                    fprintf(stderr, "gsr warning: gpu encoding is not available on your system, trying cpu encoding instead because -fallback-cpu-encoding is enabled. Install the proper vaapi drivers on your system (if supported) if you experience performance issues\n");
+                    force_cpu_encoding(args_parser);
+                }
                 break;
             }
             case GSR_VIDEO_CODEC_HEVC:
@@ -2779,14 +2789,14 @@ static const AVCodec* pick_video_codec(gsr_egl *egl, args_parser *args_parser, b
             case GSR_VIDEO_CODEC_HEVC_10BIT: {
                 fprintf(stderr, "gsr warning: selected video codec hevc is not supported, trying h264 instead\n");
                 args_parser->video_codec = GSR_VIDEO_CODEC_H264;
-                break;
+                return pick_video_codec(egl, args_parser, true, low_power, supported_video_codecs);
             }
             case GSR_VIDEO_CODEC_AV1:
             case GSR_VIDEO_CODEC_AV1_HDR:
             case GSR_VIDEO_CODEC_AV1_10BIT: {
                 fprintf(stderr, "gsr warning: selected video codec av1 is not supported, trying h264 instead\n");
                 args_parser->video_codec = GSR_VIDEO_CODEC_H264;
-                break;
+                return pick_video_codec(egl, args_parser, true, low_power, supported_video_codecs);
             }
             case GSR_VIDEO_CODEC_VP8:
             case GSR_VIDEO_CODEC_VP9:
@@ -2801,7 +2811,7 @@ static const AVCodec* pick_video_codec(gsr_egl *egl, args_parser *args_parser, b
                     print_codec_error(args_parser->video_codec);
                     _exit(11);
                 }
-                break;
+                return pick_video_codec(egl, args_parser, true, low_power, supported_video_codecs);
             }
             case GSR_VIDEO_CODEC_HEVC_VULKAN: {
                 fprintf(stderr, "gsr warning: selected video codec hevc_vulkan is not supported, trying hevc instead\n");
@@ -2812,7 +2822,7 @@ static const AVCodec* pick_video_codec(gsr_egl *egl, args_parser *args_parser, b
                     print_codec_error(args_parser->video_codec);
                     _exit(11);
                 }
-                break;
+                return pick_video_codec(egl, args_parser, true, low_power, supported_video_codecs);
             }
         }
 
@@ -2852,18 +2862,8 @@ static gsr_video_codec select_appropriate_video_codec_automatically(gsr_capture_
 
 static const AVCodec* select_video_codec_with_fallback(gsr_capture_metadata capture_metadata, args_parser *args_parser, const char *file_extension, gsr_egl *egl, bool *low_power) {
     gsr_supported_video_codecs supported_video_codecs;
-    if(!get_supported_video_codecs_with_cpu_fallback(egl, args_parser, true, &supported_video_codecs)) {
+    get_supported_video_codecs(egl, args_parser->video_codec, args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU, true, &supported_video_codecs);
-        fprintf(stderr, "gsr error: failed to query for supported video codecs\n");
+    // TODO: Use gsr_supported_video_codecs *supported_video_codecs_vulkan here to properly query vulkan video support
-        print_codec_error(args_parser->video_codec);
-        _exit(11);
-    }
-
-    if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU && args_parser->video_codec != (gsr_video_codec)GSR_VIDEO_CODEC_AUTO && args_parser->video_codec != GSR_VIDEO_CODEC_H264) {
-        fprintf(stderr, "gsr warning: cpu encoding is used but video codec isn't set to h264. Forcing video codec to h264\n");
-        args_parser->video_codec = GSR_VIDEO_CODEC_H264;
-    }
-
-    if(args_parser->video_encoder != GSR_VIDEO_ENCODER_HW_CPU)
     set_supported_video_codecs_ffmpeg(&supported_video_codecs, nullptr, egl->gpu_info.vendor);
 
     const bool video_codec_auto = args_parser->video_codec == (gsr_video_codec)GSR_VIDEO_CODEC_AUTO;
@@ -2874,33 +2874,35 @@ static const AVCodec* select_video_codec_with_fallback(gsr_capture_metadata capt
         } else if(args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU) {
             fprintf(stderr, "gsr info: using h264 encoder because a codec was not specified\n");
             args_parser->video_codec = GSR_VIDEO_CODEC_H264;
-        } else {
+        } else if(args_parser->video_encoder != GSR_VIDEO_ENCODER_HW_CPU) {
             args_parser->video_codec = select_appropriate_video_codec_automatically(capture_metadata, &supported_video_codecs);
             if(args_parser->video_codec == (gsr_video_codec)-1) {
-                fprintf(stderr, "gsr error: no video encoder was specified and neither h264, hevc nor av1 are supported on your system or you are trying to capture at a resolution higher than your system supports for each codec\n");
+                if(args_parser->fallback_cpu_encoding) {
+                    fprintf(stderr, "gsr warning: gpu encoding is not available on your system or your gpu doesn't support recording at the resolution you are trying to record, trying cpu encoding instead because -fallback-cpu-encoding is enabled. Install the proper vaapi drivers on your system (if supported) if you experience performance issues\n");
+                    force_cpu_encoding(args_parser);
+                } else {
+                    fprintf(stderr, "gsr error: no video encoder was specified and neither h264, hevc nor av1 are supported on your system or you are trying to capture at a resolution higher than your system supports for each codec.\n");
+                    fprintf(stderr, "  Ensure that you have installed the proper vaapi driver. If your gpu doesn't support video encoding then you can run gpu-screen-recorder with \"-fallback-cpu-encoding yes\" option to use cpu encoding.\n");
                     _exit(52);
                 }
             }
         }
-
+    }
-    bool use_fallback_codec = true;
 
     // TODO: Allow hevc, vp9 and av1 in (enhanced) flv (supported since ffmpeg 6.1)
     if(LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(60, 10, 100) && strcmp(file_extension, "flv") == 0) {
         if(args_parser->video_codec != GSR_VIDEO_CODEC_H264) {
             args_parser->video_codec = GSR_VIDEO_CODEC_H264;
             fprintf(stderr, "gsr warning: hevc/av1 is not compatible with flv in your outdated version of ffmpeg, falling back to h264 instead.\n");
-            use_fallback_codec = false;
         }
     } else if(strcmp(file_extension, "m3u8") == 0) {
         if(video_codec_is_av1(args_parser->video_codec)) {
             args_parser->video_codec = GSR_VIDEO_CODEC_HEVC;
             fprintf(stderr, "gsr warning: av1 is not compatible with hls (m3u8), falling back to hevc instead.\n");
-            use_fallback_codec = false;
         }
     }
 
-    const AVCodec *codec = pick_video_codec(egl, args_parser, use_fallback_codec, low_power, &supported_video_codecs);
+    const AVCodec *codec = pick_video_codec(egl, args_parser, true, low_power, &supported_video_codecs);
 
     const vec2i codec_max_resolution = codec_get_max_resolution(args_parser->video_codec, args_parser->video_encoder == GSR_VIDEO_ENCODER_HW_CPU, &supported_video_codecs);
     const vec2i capture_size = {capture_metadata.video_width, capture_metadata.video_height};
@@ -3327,6 +3329,11 @@ int main(int argc, char **argv) {
     AVStream *video_stream = nullptr;
     std::vector<AudioTrack> audio_tracks;
 
+    if(arg_parser.video_encoder == GSR_VIDEO_ENCODER_HW_CPU && arg_parser.video_codec != (gsr_video_codec)GSR_VIDEO_CODEC_AUTO && arg_parser.video_codec != GSR_VIDEO_CODEC_H264) {
+        fprintf(stderr, "gsr error: -encoder cpu was specified but a codec other than h264 was specified. -encoder cpu supports only h264 at the moment\n");
+        _exit(1);
+    }
+
     bool low_power = false;
     const AVCodec *video_codec_f = select_video_codec_with_fallback(capture_metadata, &arg_parser, file_extension.c_str(), &egl, &low_power);