From f9bf7e6c000391d19de650acb16f1afed0e6b2a3 Mon Sep 17 00:00:00 2001 From: pkv Date: Sat, 26 Aug 2023 21:40:09 +0200 Subject: [PATCH] obs-filters: Add NVIDIA Blur Filter & Background Blur This adds a Background Blur filter which blurs the background for a foreground speaker identified by NVIDIA AI Greenscreen FX. Secondly, this adds a Blur filter based on NVIDIA Video Effects, leveraging the Background Blur Filter where the mask used just sets the whole frame as background. Signed-off-by: pkv --- plugins/nv-filters/CMakeLists.txt | 2 +- plugins/nv-filters/data/locale/en-US.ini | 3 + plugins/nv-filters/data/rtx_blur.effect | 158 +++ plugins/nv-filters/nv-filters.c | 7 +- ...creen-filter.c => nvidia-videofx-filter.c} | 1253 +++++++++++------ 5 files changed, 975 insertions(+), 448 deletions(-) create mode 100644 plugins/nv-filters/data/rtx_blur.effect rename plugins/nv-filters/{nvidia-greenscreen-filter.c => nvidia-videofx-filter.c} (51%) diff --git a/plugins/nv-filters/CMakeLists.txt b/plugins/nv-filters/CMakeLists.txt index 21b9968ab..85ac94635 100644 --- a/plugins/nv-filters/CMakeLists.txt +++ b/plugins/nv-filters/CMakeLists.txt @@ -15,7 +15,7 @@ if(OS_WINDOWS) if(ENABLE_NVVFX) target_enable_feature(nv-filters "NVIDIA Video FX support" LIBNVVFX_ENABLED) - target_sources(nv-filters PRIVATE nvidia-greenscreen-filter.c) + target_sources(nv-filters PRIVATE nvidia-videofx-filter.c) else() target_disable_feature(nv-filters "NVIDIA Video FX support") endif() diff --git a/plugins/nv-filters/data/locale/en-US.ini b/plugins/nv-filters/data/locale/en-US.ini index e1da7f429..de7412f79 100644 --- a/plugins/nv-filters/data/locale/en-US.ini +++ b/plugins/nv-filters/data/locale/en-US.ini @@ -14,3 +14,6 @@ Nvvfx.Method.Greenscreen.Threshold="Threshold" Nvvfx.OutdatedSDK="WARNING: Please upgrade both NVIDIA Video & Audio SDK. Your current version of Video SDK is outdated." Nvvfx.Method.Greenscreen.Processing="Mask refresh frequency in frames" Nvvfx.Method.Greenscreen.Processing.Hint="This alleviates GPU load by generating a mask every N frames only (2 on default)." +Nvvfx.Method.BlurFilter="NVIDIA Blur Filter" +Nvvfx.Method.BackgroundBlurFilter="NVIDIA Background Blur Filter" +Nvvfx.Method.Blur.Strength="Blur Intensity" diff --git a/plugins/nv-filters/data/rtx_blur.effect b/plugins/nv-filters/data/rtx_blur.effect new file mode 100644 index 000000000..60ee7c081 --- /dev/null +++ b/plugins/nv-filters/data/rtx_blur.effect @@ -0,0 +1,158 @@ +#include "color.effect" + +uniform float4x4 ViewProj; +uniform texture2d image; +uniform float multiplier; + +uniform texture2d blurred; + +sampler_state texSampler { + Filter = Linear; + AddressU = Clamp; + AddressV = Clamp; +}; + +struct VertData { + float4 pos : POSITION; + float2 uv : TEXCOORD0; +}; + +struct VertInOut { + float2 uv : TEXCOORD0; + float4 pos : POSITION; +}; + +struct FragData { + float2 uv : TEXCOORD0; +}; + +struct FragPos { + float4 pos : POSITION; +}; + +VertInOut VSDefault(VertData v_in) +{ + VertInOut v_out; + v_out.uv = v_in.uv; + v_out.pos = mul(float4(v_in.pos.xyz, 1.), ViewProj); + return v_out; +} + +FragPos VSConvertUnorm(uint id : VERTEXID) +{ + float idHigh = float(id >> 1); + float idLow = float(id & uint(1)); + + float x = idHigh * 4.0 - 1.0; + float y = idLow * 4.0 - 1.0; + + FragPos vert_out; + vert_out.pos = float4(x, y, 0.0, 1.0); + return vert_out; +} + +float4 Mask(FragData f_in) +{ + float4 rgba = image.Sample(texSampler, f_in.uv); + rgba.rgb = max(float3(0.0, 0.0, 0.0), blurred.Sample(texSampler, f_in.uv).rgb / rgba.a); + return rgba; +} + +float4 PSMask(FragData f_in) : TARGET +{ + float4 rgba = Mask(f_in); + return rgba; +} + +float4 PSMaskMultiply(FragData f_in) : TARGET +{ + float4 rgba = Mask(f_in); + rgba.rgb *= multiplier; + return rgba; +} + +float4 PSMaskTonemap(FragData f_in) : TARGET +{ + float4 rgba = Mask(f_in); + rgba.rgb = rec709_to_rec2020(rgba.rgb); + rgba.rgb = reinhard(rgba.rgb); + rgba.rgb = rec2020_to_rec709(rgba.rgb); + return rgba; +} + +float4 PSMaskMultiplyTonemap(FragData f_in) : TARGET +{ + float4 rgba = Mask(f_in); + rgba.rgb *= multiplier; + rgba.rgb = rec709_to_rec2020(rgba.rgb); + rgba.rgb = reinhard(rgba.rgb); + rgba.rgb = rec2020_to_rec709(rgba.rgb); + return rgba; +} + +float4 PSDefault(FragPos f_in) : TARGET +{ + float4 rgba = image.Load(int3(f_in.pos.xy, 0)); + return rgba; +} + +float4 PSConvertMultiply(FragPos f_in) : TARGET +{ + float4 rgba = image.Load(int3(f_in.pos.xy, 0)); + rgba.rgb *= multiplier; + return rgba; +} + +technique Draw +{ + pass + { + vertex_shader = VSDefault(v_in); + pixel_shader = PSMask(f_in); + } +} + +technique DrawMultiply +{ + pass + { + vertex_shader = VSDefault(v_in); + pixel_shader = PSMaskMultiply(f_in); + } +} + +technique DrawTonemap +{ + pass + { + vertex_shader = VSDefault(v_in); + pixel_shader = PSMaskTonemap(f_in); + } +} + +technique DrawMultiplyTonemap +{ + pass + { + vertex_shader = VSDefault(v_in); + pixel_shader = PSMaskMultiplyTonemap(f_in); + } +} + +technique ConvertUnorm +{ + pass + { + vertex_shader = VSConvertUnorm(id); + pixel_shader = PSDefault(f_in); + } +} + +technique ConvertUnormMultiply +{ + pass + { + vertex_shader = VSConvertUnorm(id); + pixel_shader = PSConvertMultiply(f_in); + } +} diff --git a/plugins/nv-filters/nv-filters.c b/plugins/nv-filters/nv-filters.c index a9f590357..0a5fb4451 100644 --- a/plugins/nv-filters/nv-filters.c +++ b/plugins/nv-filters/nv-filters.c @@ -14,6 +14,8 @@ extern void unload_nvidia_afx(void); #endif #ifdef LIBNVVFX_ENABLED extern struct obs_source_info nvidia_greenscreen_filter_info; +extern struct obs_source_info nvidia_blur_filter_info; +extern struct obs_source_info nvidia_background_blur_filter_info; extern bool load_nvidia_vfx(void); extern void unload_nvidia_vfx(void); #endif @@ -29,8 +31,11 @@ bool obs_module_load(void) obs_enter_graphics(); const bool direct3d = gs_get_device_type() == GS_DEVICE_DIRECT3D_11; obs_leave_graphics(); - if (direct3d && load_nvidia_vfx()) + if (direct3d && load_nvidia_vfx()) { obs_register_source(&nvidia_greenscreen_filter_info); + obs_register_source(&nvidia_blur_filter_info); + obs_register_source(&nvidia_background_blur_filter_info); + } #endif return true; } diff --git a/plugins/nv-filters/nvidia-greenscreen-filter.c b/plugins/nv-filters/nvidia-videofx-filter.c similarity index 51% rename from plugins/nv-filters/nvidia-greenscreen-filter.c rename to plugins/nv-filters/nvidia-videofx-filter.c index 646c3ad5b..ad6cd646f 100644 --- a/plugins/nv-filters/nvidia-greenscreen-filter.c +++ b/plugins/nv-filters/nvidia-videofx-filter.c @@ -6,9 +6,8 @@ #include "nvvfx-load.h" /* -------------------------------------------------------- */ -#define do_log(level, format, ...) \ - blog(level, \ - "[NVIDIA AI Greenscreen (Background removal): '%s'] " format, \ +#define do_log(level, format, ...) \ + blog(level, "[NVIDIA Video Effect: '%s'] " format, \ obs_source_get_name(filter->context), ##__VA_ARGS__) #define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__) @@ -38,9 +37,18 @@ #define TEXT_PROCESSING MT_("Nvvfx.Method.Greenscreen.Processing") #define TEXT_PROCESSING_HINT MT_("Nvvfx.Method.Greenscreen.Processing.Hint") +/* Blur & background blur FX */ +#define S_STRENGTH "intensity" +#define S_STRENGTH_DEFAULT 0.5 +#define TEXT_MODE_BLUR_STRENGTH MT_("Nvvfx.Method.Blur.Strength") + +enum nvvfx_filter_id { S_FX_AIGS, S_FX_BLUR, S_FX_BG_BLUR }; + bool nvvfx_loaded = false; bool nvvfx_new_sdk = false; -struct nv_greenscreen_data { + +/* clang-format off */ +struct nvvfx_data { obs_source_t *context; bool images_allocated; bool initial_render; @@ -52,13 +60,13 @@ struct nv_greenscreen_data { /* RTX SDK vars */ NvVFX_Handle handle; - CUstream stream; // CUDA stream - int mode; // 0 = quality, 1 = performance - NvCVImage *src_img; // src img in obs format (RGBA ?) on GPU - NvCVImage *BGR_src_img; // src img in BGR on GPU - NvCVImage *A_dst_img; // mask img on GPU - NvCVImage *dst_img; // mask texture - NvCVImage *stage; // planar stage img used for transfer to texture + CUstream stream; // CUDA stream + int mode; // 0 = quality, 1 = performance + NvCVImage *src_img; // src img in obs format (RGBA ?) on GPU + NvCVImage *BGR_src_img; // src img in BGR on GPU + NvCVImage *A_dst_img; // mask img on GPU + NvCVImage *dst_img; // Greenscreen: alpha mask texture; blur: texture initialized from d3d11 (RGBA, chunky, u8) + NvCVImage *stage; // used for transfer to texture unsigned int version; NvVFX_StateObjectHandle stateObjectHandle; @@ -66,9 +74,9 @@ struct nv_greenscreen_data { gs_effect_t *effect; gs_texrender_t *render; gs_texrender_t *render_unorm; - gs_texture_t *alpha_texture; - uint32_t width; // width of texture - uint32_t height; // height of texture + gs_texture_t *alpha_texture; // either alpha or blur + uint32_t width; // width of texture + uint32_t height; // height of texture enum gs_color_space space; gs_eparam_t *mask_param; gs_eparam_t *image_param; @@ -79,10 +87,23 @@ struct nv_greenscreen_data { /* Every nth frame is processed through the FX (where n = * processing_interval) so that the same mask is used for n consecutive * frames. This is to alleviate the GPU load. Default: 1 (process every frame). + * Only used for AIGS. */ int processing_interval; int processing_counter; + + /* blur specific */ + enum nvvfx_filter_id filter_id; + NvVFX_Handle handle_blur; + CUstream stream_blur; // cuda stream + float strength; // from 0 to 1, default = 0.5 + NvCVImage *blur_BGR_dst_img; // dst img of blur FX (BGR, chunky, u8) + NvCVImage *RGBA_dst_img; // tmp img used to transfer to texture + NvCVImage *blur_dst_img; // dst img initialized from d3d11 texture (RGBA, chunky, u8) + gs_texture_t *blur_texture; + gs_eparam_t *blur_param; }; +/* clang-format on */ static const char *nv_greenscreen_filter_name(void *unused) { @@ -90,26 +111,56 @@ static const char *nv_greenscreen_filter_name(void *unused) return obs_module_text("Nvvfx.Method.Greenscreen"); } -static void nv_greenscreen_filter_update(void *data, obs_data_t *settings) +static const char *nv_blur_filter_name(void *unused) { - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; + UNUSED_PARAMETER(unused); + return obs_module_text("Nvvfx.Method.BlurFilter"); +} + +static const char *nv_background_blur_filter_name(void *unused) +{ + UNUSED_PARAMETER(unused); + return obs_module_text("Nvvfx.Method.BackgroundBlurFilter"); +} + +static void nvvfx_filter_update(void *data, obs_data_t *settings) +{ + struct nvvfx_data *filter = (struct nvvfx_data *)data; NvCV_Status vfxErr; - int mode = (int)obs_data_get_int(settings, S_MODE); - if (filter->mode != mode) { - filter->mode = mode; - vfxErr = NvVFX_SetU32(filter->handle, NVVFX_MODE, mode); - vfxErr = NvVFX_Load(filter->handle); - if (NVCV_SUCCESS != vfxErr) - error("Error loading AI Greenscreen FX %i", vfxErr); - } + enum nvvfx_fx_id id = filter->filter_id; + filter->threshold = (float)obs_data_get_double(settings, S_THRESHOLDFX); filter->processing_interval = (int)obs_data_get_int(settings, S_PROCESSING); + float strength = (float)obs_data_get_double(settings, S_STRENGTH); + if (id == S_FX_AIGS || id == S_FX_BG_BLUR) { + int mode = id == S_FX_BG_BLUR + ? (int)S_MODE_QUALITY + : (int)obs_data_get_int(settings, S_MODE); + if (filter->mode != mode) { + filter->mode = mode; + vfxErr = NvVFX_SetU32(filter->handle, NVVFX_MODE, mode); + vfxErr = NvVFX_Load(filter->handle); + if (NVCV_SUCCESS != vfxErr) + error("Error loading AI Greenscreen FX %i", + vfxErr); + } + } + if (id == S_FX_BLUR || id == S_FX_BG_BLUR) { + if (filter->strength != strength) { + filter->strength = strength; + vfxErr = NvVFX_SetF32(filter->handle_blur, + NVVFX_STRENGTH, filter->strength); + } + vfxErr = NvVFX_Load(filter->handle_blur); + if (NVCV_SUCCESS != vfxErr) + error("Error loading blur FX %i", vfxErr); + } } -static void nv_greenscreen_filter_actual_destroy(void *data) +static void nvvfx_filter_actual_destroy(void *data) { - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; + struct nvvfx_data *filter = (struct nvvfx_data *)data; if (!nvvfx_loaded) { bfree(filter); return; @@ -119,7 +170,11 @@ static void nv_greenscreen_filter_actual_destroy(void *data) if (filter->images_allocated) { obs_enter_graphics(); - gs_texture_destroy(filter->alpha_texture); + if (filter->filter_id == S_FX_AIGS) + gs_texture_destroy(filter->alpha_texture); + else + gs_texture_destroy(filter->blur_texture); + gs_texrender_destroy(filter->render); gs_texrender_destroy(filter->render_unorm); obs_leave_graphics(); @@ -128,10 +183,17 @@ static void nv_greenscreen_filter_actual_destroy(void *data) NvCVImage_Destroy(filter->A_dst_img); NvCVImage_Destroy(filter->dst_img); NvCVImage_Destroy(filter->stage); + if (filter->filter_id != S_FX_AIGS) { + NvCVImage_Destroy(filter->blur_BGR_dst_img); + NvCVImage_Destroy(filter->RGBA_dst_img); + NvCVImage_Destroy(filter->blur_dst_img); + } } - if (filter->stream) { + if (filter->stream) NvVFX_CudaStreamDestroy(filter->stream); - } + if (filter->stream_blur) + NvVFX_CudaStreamDestroy(filter->stream_blur); + if (filter->handle) { if (filter->stateObjectHandle) { NvVFX_DeallocateState(filter->handle, @@ -139,6 +201,9 @@ static void nv_greenscreen_filter_actual_destroy(void *data) } NvVFX_DestroyEffect(filter->handle); } + if (filter->handle_blur) { + NvVFX_DestroyEffect(filter->handle_blur); + } if (filter->effect) { obs_enter_graphics(); @@ -149,279 +214,94 @@ static void nv_greenscreen_filter_actual_destroy(void *data) bfree(filter); } -static void nv_greenscreen_filter_destroy(void *data) +static void nvvfx_filter_destroy(void *data) { - obs_queue_task(OBS_TASK_GRAPHICS, nv_greenscreen_filter_actual_destroy, - data, false); + obs_queue_task(OBS_TASK_GRAPHICS, nvvfx_filter_actual_destroy, data, + false); } -static void nv_greenscreen_filter_reset(void *data, calldata_t *calldata) +static void *log_nverror_destroy(struct nvvfx_data *filter, NvCV_Status vfxErr) { - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; - NvCV_Status vfxErr; + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error creating NVIDIA Video FX; error %i: %s", vfxErr, + errString); + nvvfx_filter_destroy(filter); + return NULL; +} - os_atomic_set_bool(&filter->processing_stop, true); - // first destroy - if (filter->stream) { - NvVFX_CudaStreamDestroy(filter->stream); - } - if (filter->handle) { - if (filter->stateObjectHandle) { - NvVFX_DeallocateState(filter->handle, - filter->stateObjectHandle); - } - NvVFX_DestroyEffect(filter->handle); - } - // recreate +//---------------------------------------------------------------------------// +// filter creation // + +static bool nvvfx_filter_create_internal(struct nvvfx_data *filter) +{ + NvCV_Status vfxErr; + enum nvvfx_fx_id id = filter->filter_id; /* 1. Create FX */ - vfxErr = NvVFX_CreateEffect(NVVFX_FX_GREEN_SCREEN, &filter->handle); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error recreating AI Greenscreen FX; error %i: %s", - vfxErr, errString); - nv_greenscreen_filter_destroy(filter); + switch (id) { + case S_FX_AIGS: + vfxErr = NvVFX_CreateEffect(NVVFX_FX_GREEN_SCREEN, + &filter->handle); + break; + case S_FX_BLUR: + vfxErr = NvVFX_CreateEffect(NVVFX_FX_BGBLUR, + &filter->handle_blur); + break; + case S_FX_BG_BLUR: + vfxErr = NvVFX_CreateEffect(NVVFX_FX_GREEN_SCREEN, + &filter->handle); + if (NVCV_SUCCESS != vfxErr) + log_nverror_destroy(filter, vfxErr); + vfxErr = NvVFX_CreateEffect(NVVFX_FX_BGBLUR, + &filter->handle_blur); + break; + default: + return false; } - - /* 2. Set models path & initialize CudaStream */ - char buffer[MAX_PATH]; - char modelDir[MAX_PATH]; - nvvfx_get_sdk_path(buffer, MAX_PATH); - size_t max_len = sizeof(buffer) / sizeof(char); - snprintf(modelDir, max_len, "%s\\models", buffer); - vfxErr = NvVFX_SetString(filter->handle, NVVFX_MODEL_DIRECTORY, - modelDir); - vfxErr = NvVFX_CudaStreamCreate(&filter->stream); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error creating CUDA Stream; error %i: %s", vfxErr, - errString); - nv_greenscreen_filter_destroy(filter); - } - vfxErr = NvVFX_SetCudaStream(filter->handle, NVVFX_CUDA_STREAM, - filter->stream); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error setting CUDA Stream %i", vfxErr); - nv_greenscreen_filter_destroy(filter); - } - - /* 3. load FX */ - vfxErr = NvVFX_SetU32(filter->handle, NVVFX_MODE, filter->mode); - vfxErr = NvVFX_Load(filter->handle); if (NVCV_SUCCESS != vfxErr) - error("Error loading AI Greenscreen FX %i", vfxErr); - - filter->images_allocated = false; - os_atomic_set_bool(&filter->processing_stop, false); -} - -static void init_images_greenscreen(struct nv_greenscreen_data *filter) -{ - NvCV_Status vfxErr; - uint32_t width = filter->width; - uint32_t height = filter->height; - - /* 1. create alpha texture */ - if (filter->alpha_texture) { - gs_texture_destroy(filter->alpha_texture); + log_nverror_destroy(filter, vfxErr); + /* debug */ +#ifdef _DEBUG + const char *info; + vfxErr = NvVFX_GetString(filter->handle_blur, NVVFX_INFO, &info); + blog(LOG_DEBUG, "blur fx settings /n/%s", info); +#endif + /* 2. Set models path & initialize CudaStream */ + if (id == S_FX_AIGS || id == S_FX_BG_BLUR) { + char buffer[MAX_PATH]; + char modelDir[MAX_PATH]; + nvvfx_get_sdk_path(buffer, MAX_PATH); + size_t max_len = sizeof(buffer) / sizeof(char); + snprintf(modelDir, max_len, "%s\\models", buffer); + vfxErr = NvVFX_SetString(filter->handle, NVVFX_MODEL_DIRECTORY, + modelDir); + vfxErr = NvVFX_CudaStreamCreate(&filter->stream); + if (NVCV_SUCCESS != vfxErr) + log_nverror_destroy(filter, vfxErr); + vfxErr = NvVFX_SetCudaStream(filter->handle, NVVFX_CUDA_STREAM, + filter->stream); + if (NVCV_SUCCESS != vfxErr) + log_nverror_destroy(filter, vfxErr); } - filter->alpha_texture = - gs_texture_create(width, height, GS_A8, 1, NULL, 0); - if (filter->alpha_texture == NULL) { - error("Alpha texture couldn't be created"); - goto fail; + if (id == S_FX_BLUR || id == S_FX_BG_BLUR) { + vfxErr = NvVFX_CudaStreamCreate(&filter->stream_blur); + if (NVCV_SUCCESS != vfxErr) + log_nverror_destroy(filter, vfxErr); + vfxErr = NvVFX_SetCudaStream(filter->handle_blur, + NVVFX_CUDA_STREAM, + filter->stream_blur); + if (NVCV_SUCCESS != vfxErr) + log_nverror_destroy(filter, vfxErr); } - struct ID3D11Texture2D *d11texture = - (struct ID3D11Texture2D *)gs_texture_get_obj( - filter->alpha_texture); - - /* 2. Create NvCVImage which will hold final alpha texture. */ - if (!filter->dst_img && - (NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY, - NVCV_GPU, 1, &filter->dst_img) != NVCV_SUCCESS)) { - goto fail; - } - - vfxErr = NvCVImage_InitFromD3D11Texture(filter->dst_img, d11texture); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error passing dst ID3D11Texture to img; error %i: %s", - vfxErr, errString); - goto fail; - } - - /* 3. create texrenders */ - if (filter->render) - gs_texrender_destroy(filter->render); - filter->render = gs_texrender_create( - gs_get_format_from_space(filter->space), GS_ZS_NONE); - if (!filter->render) { - error("Failed to create render texrenderer"); - goto fail; - } - if (filter->render_unorm) - gs_texrender_destroy(filter->render_unorm); - filter->render_unorm = gs_texrender_create(GS_BGRA_UNORM, GS_ZS_NONE); - if (!filter->render_unorm) { - error("Failed to create render_unorm texrenderer"); - goto fail; - } - - /* 4. Create and allocate BGR NvCVImage (fx src). */ - if (filter->BGR_src_img) { - if (NvCVImage_Realloc(filter->BGR_src_img, width, height, - NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } else { - if (NvCVImage_Create(width, height, NVCV_BGR, NVCV_U8, - NVCV_CHUNKY, NVCV_GPU, 1, - &filter->BGR_src_img) != NVCV_SUCCESS) { - goto fail; - } - if (NvCVImage_Alloc(filter->BGR_src_img, width, height, - NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } - - /* 5. Create and allocate Alpha NvCVimage (fx dst). */ - if (filter->A_dst_img) { - if (NvCVImage_Realloc(filter->A_dst_img, width, height, NVCV_A, - NVCV_U8, NVCV_CHUNKY, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } else { - if (NvCVImage_Create(width, height, NVCV_A, NVCV_U8, - NVCV_CHUNKY, NVCV_GPU, 1, - &filter->A_dst_img) != NVCV_SUCCESS) { - goto fail; - } - if (NvCVImage_Alloc(filter->A_dst_img, width, height, NVCV_A, - NVCV_U8, NVCV_CHUNKY, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } - - /* 6. Create stage NvCVImage which will be used as buffer for transfer */ - if (filter->stage) { - if (NvCVImage_Realloc(filter->stage, width, height, NVCV_RGBA, - NVCV_U8, NVCV_PLANAR, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } else { - if (NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8, - NVCV_PLANAR, NVCV_GPU, 1, - &filter->stage) != NVCV_SUCCESS) { - goto fail; - } - if (NvCVImage_Alloc(filter->stage, width, height, NVCV_RGBA, - NVCV_U8, NVCV_PLANAR, NVCV_GPU, - 1) != NVCV_SUCCESS) { - goto fail; - } - } - - /* 7. Set input & output images for nv FX. */ - if (NvVFX_SetImage(filter->handle, NVVFX_INPUT_IMAGE, - filter->BGR_src_img) != NVCV_SUCCESS) { - goto fail; - } - if (NvVFX_SetImage(filter->handle, NVVFX_OUTPUT_IMAGE, - filter->A_dst_img) != NVCV_SUCCESS) { - goto fail; - } - - filter->images_allocated = true; - return; -fail: - error("Error during allocation of images"); - os_atomic_set_bool(&filter->processing_stop, true); - return; -} - -static bool process_texture_greenscreen(struct nv_greenscreen_data *filter) -{ - /* 1. Map src img holding texture. */ - NvCV_Status vfxErr = - NvCVImage_MapResource(filter->src_img, filter->stream); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error mapping resource for source texture; error %i : %s", - vfxErr, errString); - goto fail; - } - - /* 2. Convert to BGR. */ - vfxErr = NvCVImage_Transfer(filter->src_img, filter->BGR_src_img, 1.0f, - filter->stream, filter->stage); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error converting src to BGR img; error %i: %s", vfxErr, - errString); - goto fail; - } - vfxErr = NvCVImage_UnmapResource(filter->src_img, filter->stream); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error unmapping resource for src texture; error %i: %s", - vfxErr, errString); - goto fail; - } - - /* 3. run RTX fx */ - vfxErr = NvVFX_Run(filter->handle, 1); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error running the FX; error %i: %s", vfxErr, errString); - if (vfxErr == NVCV_ERR_CUDA) - nv_greenscreen_filter_reset(filter, NULL); - } - - /* 4. Map dst texture before transfer from dst img provided by FX */ - vfxErr = NvCVImage_MapResource(filter->dst_img, filter->stream); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error mapping resource for dst texture; error %i: %s", - vfxErr, errString); - goto fail; - } - - vfxErr = NvCVImage_Transfer(filter->A_dst_img, filter->dst_img, 1.0f, - filter->stream, filter->stage); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error transferring mask to alpha texture; error %i: %s ", - vfxErr, errString); - goto fail; - } - - vfxErr = NvCVImage_UnmapResource(filter->dst_img, filter->stream); - if (vfxErr != NVCV_SUCCESS) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error unmapping resource for dst texture; error %i: %s", - vfxErr, errString); - goto fail; - } - return true; -fail: - os_atomic_set_bool(&filter->processing_stop, true); - return false; } -static void *nv_greenscreen_filter_create(obs_data_t *settings, - obs_source_t *context) +static void *nvvfx_filter_create(obs_data_t *settings, obs_source_t *context, + enum nvvfx_fx_id id) { - struct nv_greenscreen_data *filter = - (struct nv_greenscreen_data *)bzalloc(sizeof(*filter)); + struct nvvfx_data *filter = + (struct nvvfx_data *)bzalloc(sizeof(*filter)); if (!nvvfx_loaded) { - nv_greenscreen_filter_destroy(filter); + nvvfx_filter_destroy(filter); return NULL; } @@ -437,151 +317,526 @@ static void *nv_greenscreen_filter_create(obs_data_t *settings, filter->handler = NULL; filter->processing_interval = 1; filter->processing_counter = 0; + // set nvvfx_fx_id + filter->filter_id = id; + // blur specific + filter->strength = -1; - /* 1. Create FX */ - vfxErr = NvVFX_CreateEffect(NVVFX_FX_GREEN_SCREEN, &filter->handle); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error creating AI Greenscreen FX; error %i: %s", vfxErr, - errString); - nv_greenscreen_filter_destroy(filter); - return NULL; - } - - /* 2. Set models path & initialize CudaStream */ - char buffer[MAX_PATH]; - char modelDir[MAX_PATH]; - nvvfx_get_sdk_path(buffer, MAX_PATH); - size_t max_len = sizeof(buffer) / sizeof(char); - snprintf(modelDir, max_len, "%s\\models", buffer); - vfxErr = NvVFX_SetString(filter->handle, NVVFX_MODEL_DIRECTORY, - modelDir); - vfxErr = NvVFX_CudaStreamCreate(&filter->stream); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error creating CUDA Stream; error %i: %s", vfxErr, - errString); - nv_greenscreen_filter_destroy(filter); - return NULL; - } - vfxErr = NvVFX_SetCudaStream(filter->handle, NVVFX_CUDA_STREAM, - filter->stream); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = NvCV_GetErrorStringFromCode(vfxErr); - error("Error setting CUDA Stream %i", vfxErr); - nv_greenscreen_filter_destroy(filter); - return NULL; - } - /* check sdk version */ +#ifdef _DEBUG + /* sanity check of sdk version */ if (NvVFX_GetVersion(&filter->version) == NVCV_SUCCESS) { uint8_t major = (filter->version >> 24) & 0xff; uint8_t minor = (filter->version >> 16) & 0x00ff; uint8_t build = (filter->version >> 8) & 0x0000ff; uint8_t revision = (filter->version >> 0) & 0x000000ff; - // sanity check nvvfx_new_sdk = filter->version >= MIN_VFX_SDK_VERSION && nvvfx_new_sdk; } +#endif + /* 1. Create FX */ + /* 2. Set models path & initialize CudaStream */ + if (!nvvfx_filter_create_internal(filter)) + return NULL; - /* 3. Load alpha mask effect. */ - char *effect_path = obs_module_file("rtx_greenscreen.effect"); - + /* 3. Load effect. */ + char *effect_path = obs_module_file( + id != S_FX_AIGS ? "rtx_blur.effect" : "rtx_greenscreen.effect"); obs_enter_graphics(); filter->effect = gs_effect_create_from_file(effect_path, NULL); bfree(effect_path); if (filter->effect) { - filter->mask_param = - gs_effect_get_param_by_name(filter->effect, "mask"); + if (id == S_FX_AIGS) { + filter->mask_param = gs_effect_get_param_by_name( + filter->effect, "mask"); + filter->threshold_param = gs_effect_get_param_by_name( + filter->effect, "threshold"); + } else { + filter->blur_param = gs_effect_get_param_by_name( + filter->effect, "blurred"); + } filter->image_param = gs_effect_get_param_by_name(filter->effect, "image"); - filter->threshold_param = gs_effect_get_param_by_name( - filter->effect, "threshold"); + filter->multiplier_param = gs_effect_get_param_by_name( filter->effect, "multiplier"); } obs_leave_graphics(); - /* 4. Allocate state for the effect */ - if (nvvfx_new_sdk) { + /* 4. Allocate state for the AIGS & background blur */ + if (nvvfx_new_sdk && id != S_FX_BLUR) { vfxErr = NvVFX_AllocateState(filter->handle, &filter->stateObjectHandle); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = - NvCV_GetErrorStringFromCode(vfxErr); - error("Error allocating FX state %i", vfxErr); - nv_greenscreen_filter_destroy(filter); - return NULL; - } + if (NVCV_SUCCESS != vfxErr) + return log_nverror_destroy(filter, vfxErr); vfxErr = NvVFX_SetStateObjectHandleArray( filter->handle, NVVFX_STATE, &filter->stateObjectHandle); - if (NVCV_SUCCESS != vfxErr) { - const char *errString = - NvCV_GetErrorStringFromCode(vfxErr); - error("Error setting FX state %i", vfxErr); - nv_greenscreen_filter_destroy(filter); - return NULL; - } + if (NVCV_SUCCESS != vfxErr) + return log_nverror_destroy(filter, vfxErr); } if (!filter->effect) { - nv_greenscreen_filter_destroy(filter); + nvvfx_filter_destroy(filter); return NULL; } - /*---------------------------------------- */ - - nv_greenscreen_filter_update(filter, settings); + nvvfx_filter_update(filter, settings); return filter; } -static obs_properties_t *nv_greenscreen_filter_properties(void *data) +static void *nv_greenscreen_filter_create(obs_data_t *settings, + obs_source_t *context) { - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; - obs_properties_t *props = obs_properties_create(); - obs_property_t *mode = obs_properties_add_list(props, S_MODE, TEXT_MODE, - OBS_COMBO_TYPE_LIST, - OBS_COMBO_FORMAT_INT); - obs_property_list_add_int(mode, TEXT_MODE_QUALITY, S_MODE_QUALITY); - obs_property_list_add_int(mode, TEXT_MODE_PERF, S_MODE_PERF); - obs_property_t *threshold = obs_properties_add_float_slider( - props, S_THRESHOLDFX, TEXT_MODE_THRESHOLD, 0, 1, 0.05); - obs_property_t *partial = obs_properties_add_int_slider( - props, S_PROCESSING, TEXT_PROCESSING, 1, 4, 1); - obs_property_set_long_description(partial, TEXT_PROCESSING_HINT); - unsigned int version = get_lib_version(); - if (version && version < MIN_VFX_SDK_VERSION) { - obs_property_t *warning = obs_properties_add_text( - props, "deprecation", NULL, OBS_TEXT_INFO); - obs_property_text_set_info_type(warning, OBS_TEXT_INFO_WARNING); - obs_property_set_long_description(warning, TEXT_DEPRECATION); - } - - return props; + return nvvfx_filter_create(settings, context, S_FX_AIGS); } -static void nv_greenscreen_filter_defaults(obs_data_t *settings) +static void *nv_blur_filter_create(obs_data_t *settings, obs_source_t *context) { - obs_data_set_default_int(settings, S_MODE, S_MODE_QUALITY); - obs_data_set_default_double(settings, S_THRESHOLDFX, - S_THRESHOLDFX_DEFAULT); - obs_data_set_default_int(settings, S_PROCESSING, 1); + return nvvfx_filter_create(settings, context, S_FX_BLUR); +} + +static void *nv_background_blur_filter_create(obs_data_t *settings, + obs_source_t *context) +{ + return nvvfx_filter_create(settings, context, S_FX_BG_BLUR); +} + +static void nvvfx_filter_reset(void *data, calldata_t *calldata) +{ + struct nvvfx_data *filter = (struct nvvfx_data *)data; + NvCV_Status vfxErr; + + os_atomic_set_bool(&filter->processing_stop, true); + // [A] first destroy + if (filter->stream) { + NvVFX_CudaStreamDestroy(filter->stream); + } + if (filter->stream_blur) { + NvVFX_CudaStreamDestroy(filter->stream_blur); + } + if (filter->handle) { + if (filter->stateObjectHandle) { + NvVFX_DeallocateState(filter->handle, + filter->stateObjectHandle); + } + NvVFX_DestroyEffect(filter->handle); + } + if (filter->handle_blur) { + NvVFX_DestroyEffect(filter->handle_blur); + } + // [B] recreate + /* 1. Create FX */ + /* 2. Set models path & initialize CudaStream */ + if (!nvvfx_filter_create_internal(filter)) + return; + + /* 3. load FX */ + if (filter->filter_id != S_FX_BLUR) { + vfxErr = NvVFX_SetU32(filter->handle, NVVFX_MODE, filter->mode); + if (NVCV_SUCCESS != vfxErr) + error("Error loading NVIDIA Video FX %i", vfxErr); + vfxErr = NvVFX_Load(filter->handle); + if (NVCV_SUCCESS != vfxErr) + error("Error loading NVIDIA Video FX %i", vfxErr); + vfxErr = NvVFX_ResetState(filter->handle, + filter->stateObjectHandle); + } + if (filter->filter_id != S_FX_AIGS) { + vfxErr = NvVFX_SetF32(filter->handle_blur, NVVFX_STRENGTH, + filter->strength); + if (NVCV_SUCCESS != vfxErr) + error("Error loading NVIDIA Video FX %i", vfxErr); + vfxErr = NvVFX_Load(filter->handle_blur); + if (NVCV_SUCCESS != vfxErr) + error("Error loading blur FX %i", vfxErr); + } + + filter->images_allocated = false; + os_atomic_set_bool(&filter->processing_stop, false); +} + +//---------------------------------------------------------------------------// +// intialization of images // + +static bool create_alpha_texture(struct nvvfx_data *filter) +{ + NvCV_Status vfxErr; + uint32_t width = filter->width; + uint32_t height = filter->height; + + /* 1. create alpha texture */ + if (filter->alpha_texture) { + gs_texture_destroy(filter->alpha_texture); + } + filter->alpha_texture = + gs_texture_create(width, height, GS_A8, 1, NULL, 0); + if (filter->alpha_texture == NULL) { + error("Alpha texture couldn't be created"); + return false; + } + struct ID3D11Texture2D *d11texture = + (struct ID3D11Texture2D *)gs_texture_get_obj( + filter->alpha_texture); + + /* 2. Create NvCVImage which will hold final alpha texture. */ + if (!filter->dst_img) + NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->dst_img); + vfxErr = NvCVImage_InitFromD3D11Texture(filter->dst_img, d11texture); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error passing dst ID3D11Texture to img; error %i: %s", + vfxErr, errString); + return false; + } + + return true; +} + +static bool create_blur_texture(struct nvvfx_data *filter) +{ + NvCV_Status vfxErr; + uint32_t width = filter->width; + uint32_t height = filter->height; + + /* 1. Create blur texture */ + if (filter->blur_texture) { + gs_texture_destroy(filter->blur_texture); + } + filter->blur_texture = + gs_texture_create(width, height, GS_RGBA_UNORM, 1, NULL, 0); + if (filter->blur_texture == NULL) { + error("Blur texture couldn't be created"); + return false; + } + struct ID3D11Texture2D *d11texture = + (struct ID3D11Texture2D *)gs_texture_get_obj( + filter->blur_texture); + + /* 2. Create NvCVImage which will hold final blur texture */ + if (!filter->blur_dst_img) + NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->blur_dst_img); + vfxErr = NvCVImage_InitFromD3D11Texture(filter->blur_dst_img, + d11texture); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error passing dst ID3D11Texture to img; error %i: %s", + vfxErr, errString); + return false; + } + + return true; +} + +static bool create_texrenders(struct nvvfx_data *filter) +{ + if (filter->render) + gs_texrender_destroy(filter->render); + filter->render = gs_texrender_create( + gs_get_format_from_space(filter->space), GS_ZS_NONE); + if (!filter->render) { + error("Failed to create render texrenderer"); + return false; + } + if (filter->render_unorm) + gs_texrender_destroy(filter->render_unorm); + filter->render_unorm = gs_texrender_create(GS_BGRA_UNORM, GS_ZS_NONE); + if (!filter->render_unorm) { + error("Failed to create render_unorm texrenderer"); + return false; + } + + return true; +} + +static bool init_blur_images(struct nvvfx_data *filter) +{ + uint32_t width = filter->width; + uint32_t height = filter->height; + + /* 1. Create and allocate Blur BGR NvCVimage (blur FX dst) */ + NvCVImage_Create(width, height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->blur_BGR_dst_img); + NvCVImage_Alloc(filter->blur_BGR_dst_img, width, height, NVCV_BGR, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + + /* 2. Create dst NvCVImage */ + if (filter->RGBA_dst_img) { + NvCVImage_Realloc(filter->RGBA_dst_img, width, height, + NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } else { + NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->RGBA_dst_img); + NvCVImage_Alloc(filter->RGBA_dst_img, width, height, NVCV_RGBA, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } + + /* 3. Set input & output images for nv blur FX */ + NvVFX_SetImage(filter->handle_blur, NVVFX_INPUT_IMAGE, + filter->BGR_src_img); + NvVFX_SetImage(filter->handle_blur, NVVFX_INPUT_IMAGE_1, + filter->A_dst_img); + NvVFX_SetImage(filter->handle_blur, NVVFX_OUTPUT_IMAGE, + filter->blur_BGR_dst_img); + + if (NvVFX_Load(filter->handle_blur) != NVCV_SUCCESS) { + error("Error loading blur FX"); + return false; + } + + return true; +} + +static void init_images(struct nvvfx_data *filter) +{ + NvCV_Status vfxErr; + uint32_t width = filter->width; + uint32_t height = filter->height; + + /* 1. Create alpha texture & associated NvCVImage */ + if (filter->filter_id == S_FX_BG_BLUR || + filter->filter_id == S_FX_AIGS) { + if (!create_alpha_texture(filter)) + goto fail; + } + /* 2. Create blur texture & associated NvCVImage */ + if (filter->filter_id == S_FX_BG_BLUR || + filter->filter_id == S_FX_BLUR) { + if (!create_blur_texture(filter)) + goto fail; + } + /* 3. Create texrenders */ + if (!create_texrenders(filter)) + goto fail; + + /* 4. Create and allocate BGR NvCVImage (alpha mask FX src) */ + if (filter->BGR_src_img) { + NvCVImage_Realloc(filter->BGR_src_img, width, height, NVCV_BGR, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } else { + NvCVImage_Create(width, height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->BGR_src_img); + NvCVImage_Alloc(filter->BGR_src_img, width, height, NVCV_BGR, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } + + /* 5. Create and allocate Alpha NvCVimage (mask fx dst). */ + if (filter->A_dst_img) { + NvCVImage_Realloc(filter->A_dst_img, width, height, NVCV_A, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } else { + NvCVImage_Create(width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY, + NVCV_GPU, 1, &filter->A_dst_img); + NvCVImage_Alloc(filter->A_dst_img, width, height, NVCV_A, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + } + + /* 6. Create stage NvCVImage which will be used as buffer for transfer */ + vfxErr = NvCVImage_Create(width, height, NVCV_RGBA, NVCV_U8, + NVCV_CHUNKY, NVCV_GPU, 1, &filter->stage); + vfxErr = NvCVImage_Alloc(filter->stage, width, height, NVCV_RGBA, + NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + if (vfxErr != NVCV_SUCCESS) { + goto fail; + } + + /* 7. Init blur images */ + if (filter->filter_id == S_FX_BLUR || + filter->filter_id == S_FX_BG_BLUR) { + if (!init_blur_images(filter)) + goto fail; + } + + /* 8. Pass settings for AIGS (AI Greenscreen) FX */ + if (filter->filter_id == S_FX_BG_BLUR || + filter->filter_id == S_FX_AIGS) { + NvVFX_SetImage(filter->handle, NVVFX_INPUT_IMAGE, + filter->BGR_src_img); + NvVFX_SetImage(filter->handle, NVVFX_OUTPUT_IMAGE, + filter->A_dst_img); + if (filter->width) + NvVFX_SetU32(filter->handle, NVVFX_MAX_INPUT_WIDTH, + filter->width); + if (filter->height) + NvVFX_SetU32(filter->handle, NVVFX_MAX_INPUT_HEIGHT, + filter->height); + vfxErr = NvVFX_Load(filter->handle); + if (NVCV_SUCCESS != vfxErr) + error("Error loading AI Greenscreen FX %i", vfxErr); + } + filter->images_allocated = true; + return; +fail: + error("Error during allocation of images"); + os_atomic_set_bool(&filter->processing_stop, true); + return; +} + +//---------------------------------------------------------------------------// +// video processing functions // + +static bool process_texture(struct nvvfx_data *filter) +{ + enum nvvfx_fx_id id = filter->filter_id; + CUstream process_stream; + + /* 1. Map src img holding texture. */ + switch (id) { + case S_FX_AIGS: + case S_FX_BG_BLUR: + process_stream = filter->stream; + break; + case S_FX_BLUR: + process_stream = filter->stream_blur; + break; + default: + process_stream = NULL; + } + NvCV_Status vfxErr = + NvCVImage_MapResource(filter->src_img, process_stream); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error mapping resource for source texture; error %i : %s", + vfxErr, errString); + goto fail; + } + + /* 2. Convert to BGR. */ + vfxErr = NvCVImage_Transfer(filter->src_img, filter->BGR_src_img, 1.0f, + filter->stream_blur, filter->stage); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error converting src to BGR img; error %i: %s", vfxErr, + errString); + goto fail; + } + vfxErr = NvCVImage_UnmapResource(filter->src_img, process_stream); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = NvCV_GetErrorStringFromCode(vfxErr); + error("Error unmapping resource for src texture; error %i: %s", + vfxErr, errString); + goto fail; + } + + /* 3. Run AIGS (AI Greenscreen) fx */ + if (id != S_FX_BLUR) { + vfxErr = NvVFX_Run(filter->handle, 1); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error running the AIGS FX; error %i: %s", vfxErr, + errString); + if (vfxErr == NVCV_ERR_CUDA) + nvvfx_filter_reset(filter, NULL); + } + } + + if (id != S_FX_AIGS) { + /* 4. BLUR FX */ + /* 4a. Run BLUR FX except for AIGS */ + vfxErr = NvVFX_Run(filter->handle_blur, 1); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error running the BLUR FX; error %i: %s", vfxErr, + errString); + if (vfxErr == NVCV_ERR_CUDA) + nvvfx_filter_reset(filter, NULL); + } + /* 4b. Transfer blur result to an intermediate dst [RGBA, chunky, u8] */ + vfxErr = NvCVImage_Transfer(filter->blur_BGR_dst_img, + filter->RGBA_dst_img, 1.0f, + filter->stream_blur, filter->stage); + if (vfxErr != NVCV_SUCCESS) { + error("Error transferring blurred to intermediate img [RGBA, chunky, u8], error %i, ", + vfxErr); + goto fail; + } + /* 5. Map blur dst texture before transfer from dst img provided by FX */ + vfxErr = NvCVImage_MapResource(filter->blur_dst_img, + filter->stream_blur); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error mapping resource for dst texture; error %i: %s", + vfxErr, errString); + goto fail; + } + + vfxErr = NvCVImage_Transfer(filter->RGBA_dst_img, + filter->blur_dst_img, 1.0f, + filter->stream_blur, filter->stage); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error transferring mask to alpha texture; error %i: %s ", + vfxErr, errString); + goto fail; + } + + vfxErr = NvCVImage_UnmapResource(filter->blur_dst_img, + filter->stream_blur); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error unmapping resource for dst texture; error %i: %s", + vfxErr, errString); + goto fail; + } + } else { + /* 4. Map dst texture before transfer from dst img provided by AIGS FX */ + vfxErr = NvCVImage_MapResource(filter->dst_img, filter->stream); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error mapping resource for dst texture; error %i: %s", + vfxErr, errString); + goto fail; + } + + vfxErr = NvCVImage_Transfer(filter->A_dst_img, filter->dst_img, + 1.0f, filter->stream, + filter->stage); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error transferring mask to alpha texture; error %i: %s ", + vfxErr, errString); + goto fail; + } + + vfxErr = NvCVImage_UnmapResource(filter->dst_img, + filter->stream); + if (vfxErr != NVCV_SUCCESS) { + const char *errString = + NvCV_GetErrorStringFromCode(vfxErr); + error("Error unmapping resource for dst texture; error %i: %s", + vfxErr, errString); + goto fail; + } + } + + return true; +fail: + os_atomic_set_bool(&filter->processing_stop, true); + return false; } static struct obs_source_frame * -nv_greenscreen_filter_video(void *data, struct obs_source_frame *frame) +nvvfx_filter_video(void *data, struct obs_source_frame *frame) { - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; + struct nvvfx_data *filter = (struct nvvfx_data *)data; filter->got_new_frame = true; return frame; } -static void nv_greenscreen_filter_tick(void *data, float t) +static void nvvfx_filter_tick(void *data, float t) { UNUSED_PARAMETER(t); - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; + struct nvvfx_data *filter = (struct nvvfx_data *)data; if (filter->processing_stop) { return; @@ -603,11 +858,14 @@ static void nv_greenscreen_filter_tick(void *data, float t) } /* minimum size supported by SDK is (512,288) */ - filter->target_valid = cx >= 512 && cy >= 288; - if (!filter->target_valid) { - error("Size must be larger than (512,288)"); - return; + if (filter->filter_id != S_FX_BLUR) { + filter->target_valid = cx >= 512 && cy >= 288; + if (!filter->target_valid) { + error("Size must be larger than (512,288)"); + return; + } } + if (cx != filter->width && cy != filter->height) { filter->images_allocated = false; filter->width = cx; @@ -615,7 +873,7 @@ static void nv_greenscreen_filter_tick(void *data, float t) } if (!filter->images_allocated) { obs_enter_graphics(); - init_images_greenscreen(filter); + init_images(filter); obs_leave_graphics(); filter->initial_render = false; } @@ -666,9 +924,8 @@ get_tech_name_and_multiplier(enum gs_color_space current_space, return tech_name; } -static void draw_greenscreen(struct nv_greenscreen_data *filter) +static void draw_greenscreen_blur(struct nvvfx_data *filter, bool has_blur) { - /* Render alpha mask */ const enum gs_color_space source_space = filter->space; float multiplier; const char *technique = get_tech_name_and_multiplier( @@ -678,12 +935,19 @@ static void draw_greenscreen(struct nv_greenscreen_data *filter) if (obs_source_process_filter_begin_with_color_space( filter->context, format, source_space, OBS_ALLOW_DIRECT_RENDERING)) { - gs_effect_set_texture(filter->mask_param, - filter->alpha_texture); + if (has_blur) { + gs_effect_set_texture_srgb(filter->blur_param, + filter->blur_texture); + } else { + gs_effect_set_texture(filter->mask_param, + filter->alpha_texture); + gs_effect_set_float(filter->threshold_param, + filter->threshold); + } gs_effect_set_texture_srgb( filter->image_param, gs_texrender_get_texture(filter->render)); - gs_effect_set_float(filter->threshold_param, filter->threshold); + gs_effect_set_float(filter->multiplier_param, multiplier); gs_blend_state_push(); @@ -696,12 +960,12 @@ static void draw_greenscreen(struct nv_greenscreen_data *filter) } } -static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) +static void nvvfx_filter_render(void *data, gs_effect_t *effect, bool has_blur) { NvCV_Status vfxErr; - struct nv_greenscreen_data *filter = (struct nv_greenscreen_data *)data; + struct nvvfx_data *filter = (struct nvvfx_data *)data; - if (filter->processing_stop) { + if (filter->processing_stop || (has_blur && filter->strength == 0)) { obs_source_skip_video_filter(filter->context); return; } @@ -717,14 +981,14 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) /* Render processed image from earlier in the frame */ if (filter->processed_frame) { - draw_greenscreen(filter); + draw_greenscreen_blur(filter, has_blur); return; } if (parent && !filter->handler) { filter->handler = obs_source_get_signal_handler(parent); signal_handler_connect(filter->handler, "update", - nv_greenscreen_filter_reset, filter); + nvvfx_filter_reset, filter); } /* 1. Render to retrieve texture. */ @@ -750,7 +1014,7 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) if (filter->space != source_space) { filter->space = source_space; - init_images_greenscreen(filter); + init_images(filter); filter->initial_render = false; } @@ -789,16 +1053,27 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) const char *tech_name = "ConvertUnorm"; float multiplier = 1.f; - switch (source_space) { - case GS_CS_709_EXTENDED: - tech_name = "ConvertUnormTonemap"; - break; - case GS_CS_709_SCRGB: - tech_name = "ConvertUnormMultiplyTonemap"; - multiplier = - 80.0f / obs_get_video_sdr_white_level(); + if (!has_blur) { + switch (source_space) { + case GS_CS_709_EXTENDED: + tech_name = "ConvertUnormTonemap"; + break; + case GS_CS_709_SCRGB: + tech_name = + "ConvertUnormMultiplyTonemap"; + multiplier = + 80.0f / + obs_get_video_sdr_white_level(); + } + } else { + switch (source_space) { + case GS_CS_709_SCRGB: + tech_name = "ConvertUnormMultiply"; + multiplier = + 80.0f / + obs_get_video_sdr_white_level(); + } } - gs_effect_set_texture_srgb( filter->image_param, gs_texrender_get_texture(render)); @@ -862,19 +1137,23 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) if (filter->initial_render && filter->images_allocated) { bool draw = true; if (!async || filter->got_new_frame) { - if (filter->processing_counter % - filter->processing_interval == - 0) { - draw = process_texture_greenscreen(filter); - filter->processing_counter = 1; + if (!has_blur) { + if (filter->processing_counter % + filter->processing_interval == + 0) { + draw = process_texture(filter); + filter->processing_counter = 1; + } else { + filter->processing_counter++; + } } else { - filter->processing_counter++; + draw = process_texture(filter); } filter->got_new_frame = false; } if (draw) { - draw_greenscreen(filter); + draw_greenscreen_blur(filter, has_blur); filter->processed_frame = true; } } else { @@ -883,6 +1162,82 @@ static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) UNUSED_PARAMETER(effect); } +static void nv_greenscreen_filter_render(void *data, gs_effect_t *effect) +{ + nvvfx_filter_render(data, effect, false); +} + +static void nv_blur_filter_render(void *data, gs_effect_t *effect) +{ + nvvfx_filter_render(data, effect, true); +} + +static enum gs_color_space +nvvfx_filter_get_color_space(void *data, size_t count, + const enum gs_color_space *preferred_spaces) +{ + const enum gs_color_space potential_spaces[] = { + GS_CS_SRGB, + GS_CS_SRGB_16F, + GS_CS_709_EXTENDED, + }; + + struct nvvfx_data *const filter = data; + const enum gs_color_space source_space = obs_source_get_color_space( + obs_filter_get_target(filter->context), + OBS_COUNTOF(potential_spaces), potential_spaces); + + enum gs_color_space space = source_space; + for (size_t i = 0; i < count; ++i) { + space = preferred_spaces[i]; + if (space == source_space) + break; + } + + return space; +} + +static obs_properties_t *nvvfx_filter_properties(void *data) +{ + struct nvvfx_data *filter = (struct nvvfx_data *)data; + obs_properties_t *props = obs_properties_create(); + if (filter->filter_id != S_FX_AIGS) { + obs_property_t *strength = obs_properties_add_float_slider( + props, S_STRENGTH, TEXT_MODE_BLUR_STRENGTH, 0, 1, 0.05); + } else { + obs_property_t *mode = obs_properties_add_list( + props, S_MODE, TEXT_MODE, OBS_COMBO_TYPE_LIST, + OBS_COMBO_FORMAT_INT); + obs_property_list_add_int(mode, TEXT_MODE_QUALITY, + S_MODE_QUALITY); + obs_property_list_add_int(mode, TEXT_MODE_PERF, S_MODE_PERF); + obs_property_t *threshold = obs_properties_add_float_slider( + props, S_THRESHOLDFX, TEXT_MODE_THRESHOLD, 0, 1, 0.05); + obs_property_t *partial = obs_properties_add_int_slider( + props, S_PROCESSING, TEXT_PROCESSING, 1, 4, 1); + obs_property_set_long_description(partial, + TEXT_PROCESSING_HINT); + } + unsigned int version = get_lib_version(); + if (version && version < MIN_VFX_SDK_VERSION) { + obs_property_t *warning = obs_properties_add_text( + props, "deprecation", NULL, OBS_TEXT_INFO); + obs_property_text_set_info_type(warning, OBS_TEXT_INFO_WARNING); + obs_property_set_long_description(warning, TEXT_DEPRECATION); + } + + return props; +} + +static void nvvfx_filter_defaults(obs_data_t *settings) +{ + obs_data_set_default_int(settings, S_MODE, S_MODE_QUALITY); + obs_data_set_default_double(settings, S_THRESHOLDFX, + S_THRESHOLDFX_DEFAULT); + obs_data_set_default_int(settings, S_PROCESSING, 1); + obs_data_set_default_double(settings, S_STRENGTH, S_STRENGTH_DEFAULT); +} + bool load_nvidia_vfx(void) { bool old_sdk_loaded = false; @@ -1031,36 +1386,10 @@ unload_everything: return false; } -#ifdef LIBNVVFX_ENABLED void unload_nvidia_vfx(void) { release_nv_vfx(); } -#endif - -static enum gs_color_space nv_greenscreen_filter_get_color_space( - void *data, size_t count, const enum gs_color_space *preferred_spaces) -{ - const enum gs_color_space potential_spaces[] = { - GS_CS_SRGB, - GS_CS_SRGB_16F, - GS_CS_709_EXTENDED, - }; - - struct nv_greenscreen_data *const filter = data; - const enum gs_color_space source_space = obs_source_get_color_space( - obs_filter_get_target(filter->context), - OBS_COUNTOF(potential_spaces), potential_spaces); - - enum gs_color_space space = source_space; - for (size_t i = 0; i < count; ++i) { - space = preferred_spaces[i]; - if (space == source_space) - break; - } - - return space; -} struct obs_source_info nvidia_greenscreen_filter_info = { .id = "nv_greenscreen_filter", @@ -1068,12 +1397,44 @@ struct obs_source_info nvidia_greenscreen_filter_info = { .output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_SRGB, .get_name = nv_greenscreen_filter_name, .create = nv_greenscreen_filter_create, - .destroy = nv_greenscreen_filter_destroy, - .get_defaults = nv_greenscreen_filter_defaults, - .get_properties = nv_greenscreen_filter_properties, - .update = nv_greenscreen_filter_update, - .filter_video = nv_greenscreen_filter_video, + .destroy = nvvfx_filter_destroy, + .get_defaults = nvvfx_filter_defaults, + .get_properties = nvvfx_filter_properties, + .update = nvvfx_filter_update, + .filter_video = nvvfx_filter_video, .video_render = nv_greenscreen_filter_render, - .video_tick = nv_greenscreen_filter_tick, - .video_get_color_space = nv_greenscreen_filter_get_color_space, + .video_tick = nvvfx_filter_tick, + .video_get_color_space = nvvfx_filter_get_color_space, +}; + +struct obs_source_info nvidia_blur_filter_info = { + .id = "nv_blur_filter", + .type = OBS_SOURCE_TYPE_FILTER, + .output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_SRGB, + .get_name = nv_blur_filter_name, + .create = nv_blur_filter_create, + .destroy = nvvfx_filter_destroy, + .get_defaults = nvvfx_filter_defaults, + .get_properties = nvvfx_filter_properties, + .update = nvvfx_filter_update, + .filter_video = nvvfx_filter_video, + .video_render = nv_blur_filter_render, + .video_tick = nvvfx_filter_tick, + .video_get_color_space = nvvfx_filter_get_color_space, +}; + +struct obs_source_info nvidia_background_blur_filter_info = { + .id = "nv_background_blur_filter", + .type = OBS_SOURCE_TYPE_FILTER, + .output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_SRGB, + .get_name = nv_background_blur_filter_name, + .create = nv_background_blur_filter_create, + .destroy = nvvfx_filter_destroy, + .get_defaults = nvvfx_filter_defaults, + .get_properties = nvvfx_filter_properties, + .update = nvvfx_filter_update, + .filter_video = nvvfx_filter_video, + .video_render = nv_blur_filter_render, + .video_tick = nvvfx_filter_tick, + .video_get_color_space = nvvfx_filter_get_color_space, };