Update for ffmpeg 8: remove hevc vaapi padding (black bars), fix garbage output (workaround ffmpeg 8 issue)

README.md

@@ -202,7 +202,9 @@ Browsers and discord don't support hevc video codec at the moment. Choose h264 v
 Note that websites such as youtube support hevc so there is no need to choose h264 video codec if you intend to upload the video to youtube or if you want to play the video locally or if you intend to
 edit the video with a video editor. Hevc allows for better video quality (especially at lower file sizes) so hevc (or av1) is recommended for source videos.
 ## I get a black bar/distorted colors on the sides in the video
-This is mostly an issue on AMD. For av1 it's a hardware issue, see: https://gitlab.freedesktop.org/mesa/mesa/-/issues/9185. For hevc it's a software issue in the AMD driver that hasn't been fixed yet. This issue happens at certain video resolutions. If you get this issue then a workaround is to record with h264 video codec instead (using the -k h264 option).
+This is mostly an issue on AMD. For av1 it's a hardware issue, see: https://gitlab.freedesktop.org/mesa/mesa/-/issues/9185. For hevc it's a software issue in ffmpeg that was fixed in ffmpeg version 8.\
+If your ffmpeg version is older than 8 then you can use the flatpak version of GPU Screen Recorder which comes with ffmpeg version >= 8.\
+Alternatively you can record with h264 codec (-k h264, which is also the default codec) to workaround this issue.
 ## The video doesn't display or has a green/yellow overlay
 This can happen if your video player is missing the H264/HEVC video codecs. Either install the codecs or use mpv.
 ## I get stutter in the video

TODO

@@ -189,13 +189,6 @@ Use wayland color management protocol when it's available: https://gitlab.freede
 Use different exit codes for different errors. Use one for invalid -w option, another one for invalid -a option for audio devices, etc. This is to make UI error reporting better.
     Document these exit codes in an exit code .md file, or finally create a manpage where this can be documented.
 
-Ffmpeg fixed black bars in videos on amd when using hevc and when recording at some resolutions, such as 1080p:
-    https://github.com/FFmpeg/FFmpeg/commit/bcfbf2bac8f9eeeedc407b40596f5c7aaa0d5b47
-    https://github.com/FFmpeg/FFmpeg/commit/d0facac679faf45d3356dff2e2cb382580d7a521
-    Disable gpu screen recorder black bar handling when using hevc on amd when the libavcodec version is the one that comes after those commits.
-    Also consider the mesa version, to see if the gpu supports this.
-    The version is libavcodec >= 61.28.100
-
 Use opengl compute shader instead of graphics shader. This might allow for better performance when games are using 100% of graphics unit which might fix issue with 100% gpu usage causing gpu screen recorder to run slow when not using vaapi to convert rgb to nv12(?).
 
 Always disable prime run/dri prime and list all monitors to record from from all cards.

src/args_parser.c

@@ -293,8 +293,9 @@ static void usage_full() {
     printf("        Works only if your have \"Coolbits\" set to \"12\" in NVIDIA X settings, see README for more information. Note! use at your own risk! Optional, disabled by default.\n");
     printf("\n");
     printf("  -fm   Framerate mode. Should be either 'cfr' (constant frame rate), 'vfr' (variable frame rate) or 'content'. Optional, set to 'vfr' by default.\n");
-    printf("        'vfr' is recommended for recording for less issue with very high system load but some applications such as video editors may not support it properly.\n");
+    printf("        'vfr' is recommended for general recording for less issue with very high system load but some applications such as video editors may not support it properly.\n");
     printf("        'content' is currently only supported on X11 or when using portal capture option. The 'content' option matches the recording frame rate to the captured content.\n");
+    printf("        'content' is the best option to use when possible as it avoids capturing duplicate frames, which results in a smoother video.\n");
     printf("\n");
     printf("  -bm   Bitrate mode. Should be either 'auto', 'qp' (constant quality), 'vbr' (variable bitrate) or 'cbr' (constant bitrate). Optional, set to 'auto' by default which defaults to 'qp' on all devices\n");
     printf("        except steam deck that has broken drivers and doesn't support qp.\n");

src/encoder/video/vaapi.c

@@ -6,6 +6,8 @@
 #include <libavutil/hwcontext_vaapi.h>
 #include <libavutil/intreadwrite.h>
 
+#include <va/va.h>
+#include <va/va_drm.h>
 #include <va/va_drmcommon.h>
 
 #include <stdlib.h>
@@ -150,13 +152,100 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
 
 static void gsr_video_encoder_vaapi_stop(gsr_video_encoder_vaapi *self, AVCodecContext *video_codec_context);
 
+static bool supports_hevc_without_padding(const char *card_path) {
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 28, 100) && VA_CHECK_VERSION(1, 21, 0)
+    VAStatus va_status;
+    VAConfigID va_config = 0;
+    unsigned int num_surface_attr = 0;
+    VASurfaceAttrib *surface_attr_list = NULL;
+    bool supports_surface_attrib_alignment_size = false;
+    int va_major = 0;
+    int va_minor = 0;
+    bool initialized = false;
+
+    char render_path[128];
+    if(!gsr_card_path_get_render_path(card_path, render_path)) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to get /dev/dri/renderDXXX file from %s\n", card_path);
+        return false;
+    }
+
+    const int drm_fd = open(render_path, O_RDWR);
+    if(drm_fd == -1) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to open device %s\n", render_path);
+        return false;
+    }
+
+    const VADisplay va_dpy = vaGetDisplayDRM(drm_fd);
+    if(!va_dpy) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to get vaapi display for device %s\n", render_path);
+        goto done;
+    }
+
+    if(vaInitialize(va_dpy, &va_major, &va_minor) != VA_STATUS_SUCCESS) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: vaInitialize failed\n");
+        goto done;
+    }
+    initialized = true;
+
+    va_status = vaCreateConfig(va_dpy, VAProfileHEVCMain, VAEntrypointEncSlice, NULL, 0, &va_config);
+    if(va_status != VA_STATUS_SUCCESS) {
+        va_status = vaCreateConfig(va_dpy, VAProfileHEVCMain, VAEntrypointEncSliceLP, NULL, 0, &va_config);
+        if(va_status != VA_STATUS_SUCCESS) {
+            fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to create hevc vaapi config, error: %s (%d)\n", vaErrorStr(va_status), va_status);
+            return false;
+        }
+    }
+
+    va_status = vaQuerySurfaceAttributes(va_dpy, va_config, 0, &num_surface_attr);
+    if(va_status != VA_STATUS_SUCCESS) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to query vaapi surface attributes size, error: %s (%d)\n", vaErrorStr(va_status), va_status);
+        goto done;
+    }
+
+    surface_attr_list = malloc(num_surface_attr * sizeof(VASurfaceAttrib));
+    if(!surface_attr_list) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to allocate memory for %u vaapi surface attributes, error: %s (%d)\n", num_surface_attr, vaErrorStr(va_status), va_status);
+        goto done;
+    }
+
+    va_status = vaQuerySurfaceAttributes(va_dpy, va_config, surface_attr_list, &num_surface_attr);
+    if(va_status != VA_STATUS_SUCCESS) {
+        fprintf(stderr, "gsr error: supports_hevc_without_padding: failed to query vaapi surface attributes data, error: %s (%d)\n", vaErrorStr(va_status), va_status);
+        goto done;
+    }
+
+    for(unsigned int i = 0; i < num_surface_attr; ++i) {
+        if(surface_attr_list[i].type == VASurfaceAttribAlignmentSize) {
+            supports_surface_attrib_alignment_size = true;
+            break;
+        }
+    }
+
+    done:
+    free(surface_attr_list);
+    if(va_config > 0)
+        vaDestroyConfig(va_dpy, va_config);
+    if(initialized)
+        vaTerminate(va_dpy);
+    if(drm_fd > 0)
+        close(drm_fd);
+    return supports_surface_attrib_alignment_size;
+#else
+    return false;
+#endif
+}
+
 static bool gsr_video_encoder_vaapi_start(gsr_video_encoder *encoder, AVCodecContext *video_codec_context, AVFrame *frame) {
     gsr_video_encoder_vaapi *self = encoder->priv;
 
     if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
-        // TODO: dont do this if using ffmpeg reports that this is not needed (AMD driver bug that was fixed recently)
+        if(supports_hevc_without_padding(self->params.egl->card_path)) {
+            video_codec_context->width = FFALIGN(video_codec_context->width, 2);
+            video_codec_context->height = FFALIGN(video_codec_context->height, 2);
+        } else {
             video_codec_context->width = FFALIGN(video_codec_context->width, 64);
             video_codec_context->height = FFALIGN(video_codec_context->height, 16);
+        }
     } else if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_AV1) {
         // TODO: Dont do this for VCN 5 and forward which should fix this hardware bug
         video_codec_context->width = FFALIGN(video_codec_context->width, 64);

src/main.cpp

@@ -380,8 +380,9 @@ static AVCodecContext *create_video_codec_context(AVPixelFormat pix_fmt, const A
         codec_context->colorspace = AVCOL_SPC_BT709;
     }
     //codec_context->chroma_sample_location = AVCHROMA_LOC_CENTER;
-    if(codec->id == AV_CODEC_ID_HEVC)
-        codec_context->codec_tag = MKTAG('h', 'v', 'c', '1'); // QuickTime on MacOS requires this or the video wont be playable
+    // Can't use this because it's fucking broken in ffmpeg 8 or new mesa. It produces garbage output
+    //if(codec->id == AV_CODEC_ID_HEVC)
+    //    codec_context->codec_tag = MKTAG('h', 'v', 'c', '1'); // QuickTime on MacOS requires this or the video wont be playable
 
     if(arg_parser.bitrate_mode == GSR_BITRATE_MODE_CBR) {
         codec_context->bit_rate = arg_parser.video_bitrate;