obs-ffmpeg: Use Libva in FFmpeg VA-API

Libva is directly used to check if DRI devices support H264 encoding.

cmake/Modules/FindLibva.cmake

@@ -0,0 +1,49 @@
+# * Try to find Libva, once done this will define
+#
+# * LIBVA_FOUND - system has Libva
+# * LIBVA_INCLUDE_DIRS - the Libva include directory
+# * LIBVA_LIBRARIES - the libraries needed to use Libva
+# * LIBVA_DEFINITIONS - Compiler switches required for using Libva
+
+# Use pkg-config to get the directories and then use these values in the
+# find_path() and find_library() calls
+
+find_package(PkgConfig QUIET)
+if(PKG_CONFIG_FOUND)
+  pkg_check_modules(_LIBVA libva)
+endif()
+
+find_path(
+  LIBVA_INCLUDE_DIR
+  NAMES va.h
+  HINTS ${_LIBVA_INCLUDE_DIRS}
+  PATHS /usr/include /usr/local/include /opt/local/include
+  PATH_SUFFIXES va/)
+
+find_library(
+  LIBVA_LIB
+  NAMES ${_LIBVA_LIBRARIES} libva
+  HINTS ${_LIBVA_LIBRARY_DIRS}
+  PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_LIB
+                                                      LIBVA_INCLUDE_DIR)
+mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB)
+
+if(LIBVA_FOUND)
+  set(LIBVA_INCLUDE_DIRS ${LIBVA_INCLUDE_DIR})
+  set(LIBVA_LIBRARIES ${LIBVA_LIB})
+
+  if(NOT TARGET Libva::va)
+    if(IS_ABSOLUTE "${LIBVA_LIBRARIES}")
+      add_library(Libva::va UNKNOWN IMPORTED)
+      set_target_properties(Libva::va PROPERTIES IMPORTED_LOCATION
+                                                 "${LIBVA_LIBRARIES}")
+    else()
+      add_library(Libva::va INTERFACE IMPORTED)
+      set_target_properties(Libva::va PROPERTIES IMPORTED_LIBNAME
+                                                 "${LIBVA_LIBRARIES}")
+    endif()
+  endif()
+endif()

plugins/obs-ffmpeg/CMakeLists.txt

@@ -119,9 +119,11 @@ if(OS_WINDOWS)
             obs-ffmpeg.rc)
 
 elseif(OS_POSIX AND NOT OS_MACOS)
+  find_package(Libva REQUIRED)
   find_package(Libpci REQUIRED)
-  target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c)
-  target_link_libraries(obs-ffmpeg PRIVATE LIBPCI::LIBPCI)
+  target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c
+                                    vaapi-utils.h)
+  target_link_libraries(obs-ffmpeg PRIVATE Libva::va LIBPCI::LIBPCI)
 endif()
 
 setup_plugin_target(obs-ffmpeg)

plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c

@@ -38,10 +38,11 @@
 
 #include <pci/pci.h>
 
+#include "vaapi-utils.h"
 #include "obs-ffmpeg-formats.h"
 
 #define do_log(level, format, ...)                          \
-	blog(level, "[FFMPEG VAAPI encoder: '%s'] " format, \
+	blog(level, "[FFmpeg VAAPI encoder: '%s'] " format, \
 	     obs_encoder_get_name(enc->encoder), ##__VA_ARGS__)
 
 #define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__)
@@ -77,7 +78,7 @@ struct vaapi_encoder {
 static const char *vaapi_getname(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return "FFMPEG VAAPI H.264";
+	return "FFmpeg VAAPI H.264";
 }
 
 static inline bool valid_format(enum video_format format)
@@ -511,8 +512,9 @@ static void set_visible(obs_properties_t *ppts, const char *name, bool visible)
 
 static void vaapi_defaults(obs_data_t *settings)
 {
-	obs_data_set_default_string(settings, "vaapi_device",
-				    "/dev/dri/renderD128");
+	const char *device = vaapi_get_h264_default_device();
+
+	obs_data_set_default_string(settings, "vaapi_device", device);
 	obs_data_set_default_int(settings, "profile",
 				 FF_PROFILE_H264_CONSTRAINED_BASELINE);
 	obs_data_set_default_int(settings, "level", 40);
@@ -520,9 +522,67 @@ static void vaapi_defaults(obs_data_t *settings)
 	obs_data_set_default_int(settings, "keyint_sec", 0);
 	obs_data_set_default_int(settings, "bf", 0);
 	obs_data_set_default_int(settings, "rendermode", 0);
-	obs_data_set_default_string(settings, "rate_control", "CBR");
 	obs_data_set_default_int(settings, "qp", 20);
 	obs_data_set_default_int(settings, "maxrate", 0);
+
+	int drm_fd = -1;
+	VADisplay va_dpy = vaapi_open_device(&drm_fd, device, "vaapi_defaults");
+	if (!va_dpy)
+		return;
+
+	if (vaapi_device_rc_supported(VAProfileH264ConstrainedBaseline, va_dpy,
+				      VA_RC_CBR, device))
+		obs_data_set_default_string(settings, "rate_control", "CBR");
+	else if (vaapi_device_rc_supported(VAProfileH264ConstrainedBaseline,
+					   va_dpy, VA_RC_VBR, device))
+		obs_data_set_default_string(settings, "rate_control", "VBR");
+	else
+		obs_data_set_default_string(settings, "rate_control", "CQP");
+
+	vaapi_close_device(&drm_fd, va_dpy);
+}
+
+static bool vaapi_device_modified(obs_properties_t *ppts, obs_property_t *p,
+				  obs_data_t *settings)
+{
+	UNUSED_PARAMETER(p);
+
+	const char *device = obs_data_get_string(settings, "vaapi_device");
+	int drm_fd = -1;
+	VADisplay va_dpy =
+		vaapi_open_device(&drm_fd, device, "vaapi_device_modified");
+	int profile = obs_data_get_int(settings, "profile");
+	obs_property_t *rc_p = obs_properties_get(ppts, "rate_control");
+
+	obs_property_list_clear(rc_p);
+
+	if (!va_dpy || !vaapi_display_h264_supported(va_dpy, device))
+		goto fail;
+
+	switch (profile) {
+	case FF_PROFILE_H264_CONSTRAINED_BASELINE:
+		profile = VAProfileH264ConstrainedBaseline;
+		break;
+	case FF_PROFILE_H264_MAIN:
+		profile = VAProfileH264Main;
+		break;
+	case FF_PROFILE_H264_HIGH:
+		profile = VAProfileH264High;
+		break;
+	}
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CBR, device))
+		obs_property_list_add_string(rc_p, "CBR (default)", "CBR");
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_VBR, device))
+		obs_property_list_add_string(rc_p, "VBR", "VBR");
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CQP, device))
+		obs_property_list_add_string(rc_p, "CQP", "CQP");
+
+fail:
+	vaapi_close_device(&drm_fd, va_dpy);
+	return true;
 }
 
 static bool rate_control_modified(obs_properties_t *ppts, obs_property_t *p,
@@ -617,6 +677,10 @@ static obs_properties_t *vaapi_properties(void *unused)
 				bool name_found = get_device_name_from_pci(
 					pacc, pci_slot, namebuf,
 					sizeof(namebuf));
+
+				if (!vaapi_device_h264_supported(path))
+					continue;
+
 				if (!name_found)
 					obs_property_list_add_string(list, path,
 								     path);
@@ -640,6 +704,10 @@ static obs_properties_t *vaapi_properties(void *unused)
 					blog(LOG_DEBUG,
 					     "obs-ffmpeg-vaapi: A format truncation may have occurred."
 					     " This can be ignored since it is quite improbable.");
+
+				if (!vaapi_device_h264_supported(path))
+					continue;
+
 				obs_property_list_add_string(list, card, path);
 			} else {
 				break;
@@ -647,6 +715,8 @@ static obs_properties_t *vaapi_properties(void *unused)
 		}
 	}
 
+	obs_property_set_modified_callback(list, vaapi_device_modified);
+
 	list = obs_properties_add_list(props, "profile",
 				       obs_module_text("Profile"),
 				       OBS_COMBO_TYPE_LIST,
@@ -656,6 +726,8 @@ static obs_properties_t *vaapi_properties(void *unused)
 	obs_property_list_add_int(list, "Main", FF_PROFILE_H264_MAIN);
 	obs_property_list_add_int(list, "High", FF_PROFILE_H264_HIGH);
 
+	obs_property_set_modified_callback(list, vaapi_device_modified);
+
 	list = obs_properties_add_list(props, "level", obs_module_text("Level"),
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_INT);
@@ -674,9 +746,6 @@ static obs_properties_t *vaapi_properties(void *unused)
 				       obs_module_text("RateControl"),
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_STRING);
-	obs_property_list_add_string(list, "CBR (default)", "CBR");
-	obs_property_list_add_string(list, "CQP", "CQP");
-	obs_property_list_add_string(list, "VBR", "VBR");
 
 	obs_property_set_modified_callback(list, rate_control_modified);
 

plugins/obs-ffmpeg/obs-ffmpeg.c

@@ -14,6 +14,13 @@
 #include "jim-nvenc.h"
 #endif
 
+#if !defined(_WIN32) && !defined(__APPLE__) && \
+	LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 27, 100)
+#include "vaapi-utils.h"
+
+#define LIBAVUTIL_VAAPI_AVAILABLE
+#endif
+
 OBS_DECLARE_MODULE()
 OBS_MODULE_USE_DEFAULT_LOCALE("obs-ffmpeg", "en-US")
 MODULE_EXPORT const char *obs_module_description(void)
@@ -36,10 +43,6 @@ extern struct obs_encoder_info hevc_nvenc_encoder_info;
 extern struct obs_encoder_info svt_av1_encoder_info;
 extern struct obs_encoder_info aom_av1_encoder_info;
 
-#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 27, 100)
-#define LIBAVUTIL_VAAPI_AVAILABLE
-#endif
-
 #ifdef LIBAVUTIL_VAAPI_AVAILABLE
 extern struct obs_encoder_info vaapi_encoder_info;
 #endif
@@ -320,10 +323,16 @@ static bool nvenc_supported(bool *out_h264, bool *out_hevc, bool *out_av1)
 #endif
 
 #ifdef LIBAVUTIL_VAAPI_AVAILABLE
-static bool vaapi_supported(void)
+static bool h264_vaapi_supported(void)
 {
 	const AVCodec *vaenc = avcodec_find_encoder_by_name("h264_vaapi");
-	return !!vaenc;
+
+	if (!vaenc)
+		return false;
+
+	/* NOTE: If default device is NULL, it means there is no device
+	 * that support H264. */
+	return vaapi_get_h264_default_device() != NULL;
 }
 #endif
 
@@ -403,10 +412,18 @@ bool obs_module_load(void)
 	amf_load();
 #endif
 
-#if !defined(_WIN32) && defined(LIBAVUTIL_VAAPI_AVAILABLE)
-	if (vaapi_supported()) {
-		blog(LOG_INFO, "FFMPEG VAAPI supported");
+#ifdef LIBAVUTIL_VAAPI_AVAILABLE
+	const char *libva_env = getenv("LIBVA_DRIVER_NAME");
+	if (!!libva_env)
+		blog(LOG_WARNING,
+		     "LIBVA_DRIVER_NAME variable is set,"
+		     " this could prevent FFmpeg VAAPI from working correctly");
+
+	if (h264_vaapi_supported()) {
+		blog(LOG_INFO, "FFmpeg VAAPI H264 encoding supported");
 		obs_register_encoder(&vaapi_encoder_info);
+	} else {
+		blog(LOG_INFO, "FFmpeg VAAPI H264 encoding not supported");
 	}
 #endif
 #endif

plugins/obs-ffmpeg/vaapi-utils.c

@@ -0,0 +1,262 @@
+// SPDX-FileCopyrightText: 2022 tytan652 <tytan652@tytanium.xyz>
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "vaapi-utils.h"
+
+#include <util/bmem.h>
+#include <util/dstr.h>
+
+#include <va/va_drm.h>
+#include <va/va_str.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+static bool version_logged = false;
+
+inline static VADisplay vaapi_open_display_drm(int *fd, const char *device_path)
+{
+	VADisplay va_dpy;
+
+	if (!device_path)
+		return NULL;
+
+	*fd = open(device_path, O_RDWR);
+	if (*fd < 0) {
+		blog(LOG_ERROR, "VAAPI: Failed to open device '%s'",
+		     device_path);
+		return NULL;
+	}
+
+	va_dpy = vaGetDisplayDRM(*fd);
+
+	if (!va_dpy) {
+		blog(LOG_ERROR, "VAAPI: Failed to initialize DRM display");
+		return NULL;
+	}
+
+	return va_dpy;
+}
+
+inline static void vaapi_close_display_drm(int *fd)
+{
+	if (*fd < 0)
+		return;
+
+	close(*fd);
+	*fd = -1;
+}
+
+static void vaapi_log_info_cb(void *user_context, const char *message)
+{
+	UNUSED_PARAMETER(user_context);
+
+	// Libva message always ends with a newline
+	struct dstr m;
+	dstr_init_copy(&m, message);
+	dstr_depad(&m);
+
+	blog(LOG_DEBUG, "Libva: %s", m.array);
+
+	dstr_free(&m);
+}
+
+static void vaapi_log_error_cb(void *user_context, const char *message)
+{
+	UNUSED_PARAMETER(user_context);
+
+	// Libva message always ends with a newline
+	struct dstr m;
+	dstr_init_copy(&m, message);
+	dstr_depad(&m);
+
+	blog(LOG_DEBUG, "Libva error: %s", m.array);
+
+	dstr_free(&m);
+}
+
+VADisplay vaapi_open_device(int *fd, const char *device_path,
+			    const char *func_name)
+{
+	VADisplay va_dpy;
+	VAStatus va_status;
+	int major, minor;
+	const char *driver;
+
+	va_dpy = vaapi_open_display_drm(fd, device_path);
+	if (!va_dpy)
+		return NULL;
+
+	blog(LOG_DEBUG, "VAAPI: Initializing display in %s", func_name);
+
+	vaSetInfoCallback(va_dpy, vaapi_log_info_cb, NULL);
+	vaSetErrorCallback(va_dpy, vaapi_log_error_cb, NULL);
+
+	va_status = vaInitialize(va_dpy, &major, &minor);
+
+	if (va_status != VA_STATUS_SUCCESS) {
+		blog(LOG_ERROR, "VAAPI: Failed to initialize display in %s",
+		     func_name);
+		return NULL;
+	}
+
+	blog(LOG_DEBUG, "VAAPI: Display initialized");
+
+	if (!version_logged) {
+		blog(LOG_INFO, "VAAPI: API version %d.%d", major, minor);
+		version_logged = true;
+	}
+
+	driver = vaQueryVendorString(va_dpy);
+
+	blog(LOG_DEBUG, "VAAPI: '%s' in use for device '%s'", driver,
+	     device_path);
+
+	return va_dpy;
+}
+
+void vaapi_close_device(int *fd, VADisplay dpy)
+{
+	vaTerminate(dpy);
+	vaapi_close_display_drm(fd);
+}
+
+static uint32_t vaapi_display_ep_combo_rate_controls(VAProfile profile,
+						     VAEntrypoint entrypoint,
+						     VADisplay dpy,
+						     const char *device_path)
+{
+	bool ret = false;
+	VAStatus va_status;
+	VAConfigAttrib attrib[1];
+	attrib->type = VAConfigAttribRateControl;
+
+	va_status = vaGetConfigAttributes(dpy, profile, entrypoint, attrib, 1);
+
+	switch (va_status) {
+	case VA_STATUS_SUCCESS:
+		return attrib->value;
+	case VA_STATUS_ERROR_UNSUPPORTED_PROFILE:
+		blog(LOG_DEBUG, "VAAPI: %s is not supported by the device '%s'",
+		     vaProfileStr(profile), device_path);
+		return 0;
+	case VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT:
+		blog(LOG_DEBUG,
+		     "VAAPI: %s %s is not supported by the device '%s'",
+		     vaProfileStr(profile), vaEntrypointStr(entrypoint),
+		     device_path);
+		return 0;
+	default:
+		blog(LOG_ERROR,
+		     "VAAPI: Fail to get RC attribute from the %s %s of the device '%s'",
+		     vaProfileStr(profile), vaEntrypointStr(entrypoint),
+		     device_path);
+		return 0;
+	}
+}
+
+static bool vaapi_display_ep_combo_supported(VAProfile profile,
+					     VAEntrypoint entrypoint,
+					     VADisplay dpy,
+					     const char *device_path)
+{
+	uint32_t ret = vaapi_display_ep_combo_rate_controls(profile, entrypoint,
+							    dpy, device_path);
+	if (ret & VA_RC_CBR || ret & VA_RC_CQP || ret & VA_RC_VBR)
+		return true;
+
+	return false;
+}
+
+bool vaapi_device_rc_supported(VAProfile profile, VADisplay dpy, uint32_t rc,
+			       const char *device_path)
+{
+	uint32_t ret = vaapi_display_ep_combo_rate_controls(
+		profile, VAEntrypointEncSlice, dpy, device_path);
+	if (ret & rc)
+		return true;
+	ret = vaapi_display_ep_combo_rate_controls(
+		profile, VAEntrypointEncSliceLP, dpy, device_path);
+	if (ret & rc)
+		return true;
+
+	return false;
+}
+
+#define CHECK_PROFILE(ret, profile, va_dpy, device_path)                      \
+	if (vaapi_display_ep_combo_supported(profile, VAEntrypointEncSlice,   \
+					     va_dpy, device_path)) {          \
+		blog(LOG_DEBUG, "'%s' support encoding with %s", device_path, \
+		     vaProfileStr(profile));                                  \
+		ret |= true;                                                  \
+	}
+
+#define CHECK_PROFILE_LP(ret, profile, va_dpy, device_path)                   \
+	if (vaapi_display_ep_combo_supported(profile, VAEntrypointEncSliceLP, \
+					     va_dpy, device_path)) {          \
+		blog(LOG_DEBUG, "'%s' support low power encoding with %s",    \
+		     device_path, vaProfileStr(profile));                     \
+		ret |= true;                                                  \
+	}
+
+bool vaapi_display_h264_supported(VADisplay dpy, const char *device_path)
+{
+	bool ret = false;
+
+	CHECK_PROFILE(ret, VAProfileH264ConstrainedBaseline, dpy, device_path);
+	CHECK_PROFILE(ret, VAProfileH264Main, dpy, device_path);
+	CHECK_PROFILE(ret, VAProfileH264High, dpy, device_path);
+
+	if (!ret) {
+		CHECK_PROFILE_LP(ret, VAProfileH264ConstrainedBaseline, dpy,
+				 device_path);
+		CHECK_PROFILE_LP(ret, VAProfileH264Main, dpy, device_path);
+		CHECK_PROFILE_LP(ret, VAProfileH264High, dpy, device_path);
+	}
+
+	return ret;
+}
+
+bool vaapi_device_h264_supported(const char *device_path)
+{
+	bool ret = false;
+	VADisplay va_dpy;
+
+	int drm_fd = -1;
+
+	va_dpy = vaapi_open_device(&drm_fd, device_path,
+				   "vaapi_device_h264_supported");
+	if (!va_dpy)
+		return false;
+
+	ret = vaapi_display_h264_supported(va_dpy, device_path);
+
+	vaapi_close_device(&drm_fd, va_dpy);
+
+	return ret;
+}
+
+const char *vaapi_get_h264_default_device()
+{
+	static const char *default_h264_device = NULL;
+
+	if (!default_h264_device) {
+		bool ret = false;
+		char path[32] = "/dev/dri/renderD1";
+		for (int i = 28;; i++) {
+			sprintf(path, "/dev/dri/renderD1%d", i);
+			if (access(path, F_OK) != 0)
+				break;
+
+			ret = vaapi_device_h264_supported(path);
+			if (ret) {
+				default_h264_device = strdup(path);
+				break;
+			}
+		}
+	}
+
+	return default_h264_device;
+}

plugins/obs-ffmpeg/vaapi-utils.h

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: 2022 tytan652 <tytan652@tytanium.xyz>
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <util/base.h>
+
+#include <va/va.h>
+
+VADisplay vaapi_open_device(int *fd, const char *device_path,
+			    const char *func_name);
+void vaapi_close_device(int *fd, VADisplay dpy);
+
+bool vaapi_device_rc_supported(VAProfile profile, VADisplay dpy, uint32_t rc,
+			       const char *device_path);
+
+bool vaapi_display_h264_supported(VADisplay dpy, const char *device_path);
+
+bool vaapi_device_h264_supported(const char *device_path);
+
+const char *vaapi_get_h264_default_device(void);