From 31eb7a3ad1fb9acb2fc2f0116a0336cbe32f92ab Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Sat, 23 Oct 2021 20:05:36 +0200 Subject: [PATCH] filter/virtual-greenscreen: Add Virtual Greenscreen filter --- CMakeLists.txt | 46 +- data/effects/virtual-greenscreen.effect | 70 ++ data/locale/en-US.ini | 9 + source/filters/filter-virtual-greenscreen.cpp | 657 ++++++++++++++++++ source/filters/filter-virtual-greenscreen.hpp | 131 ++++ source/plugin.cpp | 9 + 6 files changed, 920 insertions(+), 2 deletions(-) create mode 100644 data/effects/virtual-greenscreen.effect create mode 100644 source/filters/filter-virtual-greenscreen.cpp create mode 100644 source/filters/filter-virtual-greenscreen.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e3055284..a186b29f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -309,6 +309,8 @@ set(${PREFIX}ENABLE_FILTER_SHADER ON CACHE BOOL "Enable Shader Filter") set(${PREFIX}ENABLE_FILTER_TRANSFORM ON CACHE BOOL "Enable Transform Filter") set(${PREFIX}ENABLE_FILTER_UPSCALING ON CACHE BOOL "Enable Upscaling Filter") set(${PREFIX}ENABLE_FILTER_UPSCALING_NVIDIA ON CACHE BOOL "Enable NVIDIA provider(s) for Upscaling Filter") +set(${PREFIX}ENABLE_FILTER_VIRTUAL_GREENSCREEN ON CACHE BOOL "Enable Virtual Greenscreen Filter") +set(${PREFIX}ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA ON CACHE BOOL "Enable NVIDIA provider(s) for Virtual Greenscreen Filter") ## Sources set(${PREFIX}ENABLE_SOURCE_MIRROR ON CACHE BOOL "Enable Mirror Source") @@ -738,6 +740,26 @@ function(feature_filter_upscaling RESOLVE) endif() endfunction() +function(feature_filter_virtual_greenscreen RESOLVE) + is_feature_enabled(FILTER_VIRTUAL_GREENSCREEN T_CHECK) + if(RESOLVE AND T_CHECK) + # Verify that the requirements for the providers are available + if(NOT HAVE_NVIDIA_VFX_SDK) + message(WARNING "${LOGPREFIX}: 'NVIDIA Video Effects SDK' is missing. Disabling NVIDIA provider(s)...") + set_feature_disabled(FILTER_VIRTUAL_GREENSCREEN_NVIDIA ON) + endif() + + # Verify that we have at least one provider for Video Super-Resolution. + is_feature_enabled(FILTER_VIRTUAL_GREENSCREEN_NVIDIA T_CHECK_NVIDIA) + if (NOT T_CHECK_NVIDIA) + message(WARNING "${LOGPREFIX}: Virtual Greenscreen has no available providers. Disabling...") + set_feature_disabled(FILTER_VIRTUAL_GREENSCREEN ON) + endif() + elseif(T_CHECK) + set(REQUIRE_NVIDIA_VFX_SDK ON PARENT_SCOPE) + endif() +endfunction() + function(feature_source_mirror RESOLVE) is_feature_enabled(SOURCE_MIRROR T_CHECK) endfunction() @@ -795,6 +817,7 @@ feature_filter_sdf_effects(OFF) feature_filter_shader(OFF) feature_filter_transform(OFF) feature_filter_upscaling(OFF) +feature_filter_virtual_greenscreen(OFF) feature_source_mirror(OFF) feature_source_shader(OFF) feature_transition_shader(OFF) @@ -951,6 +974,7 @@ feature_filter_sdf_effects(ON) feature_filter_shader(ON) feature_filter_transform(ON) feature_filter_upscaling(ON) +feature_filter_virtual_greenscreen(ON) feature_source_mirror(ON) feature_source_shader(ON) feature_transition_shader(ON) @@ -1419,7 +1443,7 @@ if(T_CHECK) ) endif() -# Filter/Video Denoising +# Filter/Denoising is_feature_enabled(FILTER_DENOISING T_CHECK) if(T_CHECK) list(APPEND PROJECT_PRIVATE_SOURCE @@ -1437,7 +1461,7 @@ if(T_CHECK) endif() endif() -# Filter/Video Super-Resolution +# Filter/Upscaling is_feature_enabled(FILTER_UPSCALING T_CHECK) if(T_CHECK) list(APPEND PROJECT_PRIVATE_SOURCE @@ -1455,6 +1479,24 @@ if(T_CHECK) endif() endif() +# Filter/Virtual Greenscreen +is_feature_enabled(FILTER_VIRTUAL_GREENSCREEN T_CHECK) +if(T_CHECK) + list(APPEND PROJECT_PRIVATE_SOURCE + "source/filters/filter-virtual-greenscreen.hpp" + "source/filters/filter-virtual-greenscreen.cpp" + ) + list(APPEND PROJECT_DEFINITIONS + ENABLE_FILTER_VIRTUAL_GREENSCREEN + ) + is_feature_enabled(FILTER_VIRTUAL_GREENSCREEN_NVIDIA T_CHECK) + if (T_CHECK) + list(APPEND PROJECT_DEFINITIONS + ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + ) + endif() +endif() + # Source/Mirror is_feature_enabled(SOURCE_MIRROR T_CHECK) if(T_CHECK) diff --git a/data/effects/virtual-greenscreen.effect b/data/effects/virtual-greenscreen.effect new file mode 100644 index 00000000..182b951c --- /dev/null +++ b/data/effects/virtual-greenscreen.effect @@ -0,0 +1,70 @@ +#include "shared.effect" + +uniform texture2D InputA< + bool automatic = true; +>; +uniform texture2D InputB< + bool automatic = true; +>; +uniform float Threshold< + string name = "Threshold"; + string suffix = " %"; + float minimum = 0.; + float maximum = 100.; + float step = .01; + float scale = .01; +> = 10.; +uniform float ThresholdRange< + string name = "Threshold Range"; + string suffix = " %"; + float minimum = 0.; + float maximum = 100.; + float step = .01; + float scale = .01; +> = 10.; + +//------------------------------------------------------------------------------ +// Technique: Draw +//------------------------------------------------------------------------------ +// Parameters: +// - InputA: RGBA Texture +// - InputB: XXXA Texture + +float4 PSDrawAlpha(VertexData vtx) : TARGET { + return InputA.Sample(BlankSampler, vtx.uv); +}; + +technique DrawAlpha +{ + pass + { + vertex_shader = DefaultVertexShader(vtx); + pixel_shader = PSDrawAlpha(vtx); + }; +}; + +//------------------------------------------------------------------------------ +// Technique: Draw +//------------------------------------------------------------------------------ +// Parameters: +// - InputA: RGBX Texture +// - InputB: XXXA Texture +// - Threshold: Alha threshold to be "visible". + +float4 PSDrawAlphaThreshold(VertexData vtx) : TARGET { + float4 rgba = InputA.Sample(BlankSampler, vtx.uv); + float4 xxxa = InputB.Sample(BlankSampler, vtx.uv); + + rgba.a = smoothstep(Threshold - ThresholdRange * .5, Threshold + ThresholdRange * .5, xxxa.a); + + return rgba; +}; + +technique DrawAlphaThreshold +{ + pass + { + vertex_shader = DefaultVertexShader(vtx); + pixel_shader = PSDrawAlphaThreshold(vtx); + }; +}; diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 40479019..0b53f495 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -440,6 +440,15 @@ Filter.Upscaling.NVIDIA.SuperRes.Strength="Strength" Filter.Upscaling.NVIDIA.SuperRes.Strength.Weak="Weak" Filter.Upscaling.NVIDIA.SuperRes.Strength.Strong="Strong" +# Filter - Virtual Greenscreen +Filter.VirtualGreenscreen="Virtual Greenscreen" +Filter.VirtualGreenscreen.Provider="Provider" +Filter.VirtualGreenscreen.Provider.NVIDIA.Greenscreen="NVIDIA® Greenscreen, powered by NVIDIA® Broadcast" +Filter.VirtualGreenscreen.NVIDIA.Greenscreen="NVIDIA® Greenscreen" +Filter.VirtualGreenscreen.NVIDIA.Greenscreen.Mode="Mode" +Filter.VirtualGreenscreen.NVIDIA.Greenscreen.Mode.Performance="Performance" +Filter.VirtualGreenscreen.NVIDIA.Greenscreen.Mode.Quality="Quality" + # Source - Mirror Source.Mirror="Source Mirror" Source.Mirror.Source="Source" diff --git a/source/filters/filter-virtual-greenscreen.cpp b/source/filters/filter-virtual-greenscreen.cpp new file mode 100644 index 00000000..8ab5d0a4 --- /dev/null +++ b/source/filters/filter-virtual-greenscreen.cpp @@ -0,0 +1,657 @@ +// Copyright (c) 2020 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "filter-virtual-greenscreen.hpp" +#include +#include "obs/gs/gs-helper.hpp" +#include "plugin.hpp" +#include "util/util-logging.hpp" + +#ifdef _DEBUG +#define ST_PREFIX "<%s> " +#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#else +#define ST_PREFIX " " +#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) +#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) +#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) +#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) +#endif + +#define ST_I18N "Filter.VirtualGreenscreen" +#define ST_KEY_PROVIDER "Provider" +#define ST_I18N_PROVIDER ST_I18N "." ST_KEY_PROVIDER +#define ST_I18N_PROVIDER_NVIDIA_GREENSCREEN ST_I18N_PROVIDER ".NVIDIA.Greenscreen" + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA +#define ST_KEY_NVIDIA_GREENSCREEN "NVIDIA.Greenscreen" +#define ST_I18N_NVIDIA_GREENSCREEN ST_I18N "." ST_KEY_NVIDIA_GREENSCREEN +#define ST_KEY_NVIDIA_GREENSCREEN_MODE ST_KEY_NVIDIA_GREENSCREEN ".Mode" +#define ST_I18N_NVIDIA_GREENSCREEN_MODE ST_I18N_NVIDIA_GREENSCREEN ".Mode" +#define ST_I18N_NVIDIA_GREENSCREEN_MODE_PERFORMANCE ST_I18N_NVIDIA_GREENSCREEN_MODE ".Performance" +#define ST_I18N_NVIDIA_GREENSCREEN_MODE_QUALITY ST_I18N_NVIDIA_GREENSCREEN_MODE ".Quality" +#endif + +using streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory; +using streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance; +using streamfx::filter::virtual_greenscreen::virtual_greenscreen_provider; + +static constexpr std::string_view HELP_URL = "https://github.com/Xaymar/obs-StreamFX/wiki/Filter-Virtual-Greenscreen"; + +/** Priority of providers for automatic selection if more than one is available. + * + */ +static virtual_greenscreen_provider provider_priority[] = { + virtual_greenscreen_provider::NVIDIA_GREENSCREEN, +}; + +const char* streamfx::filter::virtual_greenscreen::cstring(virtual_greenscreen_provider provider) +{ + switch (provider) { + case virtual_greenscreen_provider::INVALID: + return "N/A"; + case virtual_greenscreen_provider::AUTOMATIC: + return D_TRANSLATE(S_STATE_AUTOMATIC); + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + return D_TRANSLATE(ST_I18N_PROVIDER_NVIDIA_GREENSCREEN); + default: + throw std::runtime_error("Missing Conversion Entry"); + } +} + +std::string streamfx::filter::virtual_greenscreen::string(virtual_greenscreen_provider provider) +{ + return cstring(provider); +} + +//------------------------------------------------------------------------------ +// Instance +//------------------------------------------------------------------------------ +virtual_greenscreen_instance::virtual_greenscreen_instance(obs_data_t* data, obs_source_t* self) + : obs::source_instance(data, self), + + _size(1, 1), _provider(virtual_greenscreen_provider::INVALID), + _provider_ui(virtual_greenscreen_provider::INVALID), _provider_ready(false), _provider_lock(), _provider_task(), + _effect(), _channel0_sampler(), _channel1_sampler(), _input(), _output_color(), _output_alpha(), _dirty(true) +{ + { + ::streamfx::obs::gs::context gctx; + + // Create the render target for the input buffering. + _input = std::make_shared<::streamfx::obs::gs::rendertarget>(GS_RGBA_UNORM, GS_ZS_NONE); + _input->render(1, 1); // Preallocate the RT on the driver and GPU. + _output_color = _input->get_texture(); + _output_alpha = _input->get_texture(); + + // Load the required effect. + { + std::filesystem::path file = ::streamfx::data_file_path("effects/virtual-greenscreen.effect"); + try { + _effect = std::make_shared<::streamfx::obs::gs::effect>(file); + } catch (...) { + D_LOG_ERROR("Failed to load '%s'.", file.generic_u8string().c_str()); + } + } + + // Create Samplers + _channel0_sampler = std::make_shared<::streamfx::obs::gs::sampler>(); + _channel0_sampler->set_filter(gs_sample_filter::GS_FILTER_LINEAR); + _channel0_sampler->set_address_mode_u(GS_ADDRESS_CLAMP); + _channel0_sampler->set_address_mode_v(GS_ADDRESS_CLAMP); + _channel1_sampler = std::make_shared<::streamfx::obs::gs::sampler>(); + _channel1_sampler->set_filter(gs_sample_filter::GS_FILTER_LINEAR); + _channel1_sampler->set_address_mode_u(GS_ADDRESS_CLAMP); + _channel1_sampler->set_address_mode_v(GS_ADDRESS_CLAMP); + } + + if (data) { + load(data); + } +} + +virtual_greenscreen_instance::~virtual_greenscreen_instance() +{ + // TODO: Make this asynchronous. + std::unique_lock ul(_provider_lock); + switch (_provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_unload(); + break; +#endif + default: + break; + } +} + +void virtual_greenscreen_instance::load(obs_data_t* data) +{ + update(data); +} + +void virtual_greenscreen_instance::migrate(obs_data_t* data, uint64_t version) {} + +void virtual_greenscreen_instance::update(obs_data_t* data) +{ + // Check if the user changed which Denoising provider we use. + virtual_greenscreen_provider provider = + static_cast(obs_data_get_int(data, ST_KEY_PROVIDER)); + if (provider == virtual_greenscreen_provider::AUTOMATIC) { + provider = virtual_greenscreen_factory::get()->find_ideal_provider(); + } + + // Check if the provider was changed, and if so switch. + if (provider != _provider) { + _provider_ui = provider; + switch_provider(provider); + } + + if (_provider_ready) { + std::unique_lock ul(_provider_lock); + + switch (_provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_update(data); + break; +#endif + default: + break; + } + } +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::properties(obs_properties_t* properties) +{ + switch (_provider_ui) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_properties(properties); + break; +#endif + default: + break; + } +} + +uint32_t streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::get_width() +{ + return std::max(_size.first, 1); +} + +uint32_t streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::get_height() +{ + return std::max(_size.second, 1); +} + +void virtual_greenscreen_instance::video_tick(float_t time) +{ + auto target = obs_filter_get_target(_self); + auto width = obs_source_get_base_width(target); + auto height = obs_source_get_base_height(target); + _size = {width, height}; + + // Allow the provider to restrict the size. + if (target && _provider_ready) { + std::unique_lock ul(_provider_lock); + + switch (_provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_size(); + break; +#endif + default: + break; + } + } + + _dirty = true; +} + +void virtual_greenscreen_instance::video_render(gs_effect_t* effect) +{ + auto parent = obs_filter_get_parent(_self); + auto target = obs_filter_get_target(_self); + auto width = obs_source_get_base_width(target); + auto height = obs_source_get_base_height(target); + vec4 blank = vec4{0, 0, 0, 0}; + + // Ensure we have the bare minimum of valid information. + target = target ? target : parent; + effect = effect ? effect : obs_get_base_effect(OBS_EFFECT_DEFAULT); + + // Skip the filter if: + // - The Provider isn't ready yet. + // - We don't have a target. + // - The width/height of the next filter in the chain is empty. + if (!_provider_ready || !target || (width == 0) || (height == 0)) { + obs_source_skip_video_filter(_self); + return; + } + +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler0{::streamfx::obs::gs::debug_color_source, + "StreamFX Virtual Green-Screen"}; + ::streamfx::obs::gs::debug_marker profiler0_0{::streamfx::obs::gs::debug_color_gray, "'%s' on '%s'", + obs_source_get_name(_self), obs_source_get_name(parent)}; +#endif + + if (_dirty) { + // Lock the provider from being changed. + std::unique_lock ul(_provider_lock); + + { // Capture the incoming frame. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_capture, "Capture"}; +#endif + if (obs_source_process_filter_begin(_self, GS_RGBA, OBS_ALLOW_DIRECT_RENDERING)) { + auto op = _input->render(_size.first, _size.second); + + // Matrix + gs_matrix_push(); + gs_ortho(0., 1., 0., 1., 0., 1.); + + // Clear the buffer + gs_clear(GS_CLEAR_COLOR | GS_CLEAR_DEPTH, &blank, 0, 0); + + // Set GPU state + gs_blend_state_push(); + gs_enable_color(true, true, true, true); + gs_enable_blending(false); + gs_enable_depth_test(false); + gs_enable_stencil_test(false); + gs_set_cull_mode(GS_NEITHER); + + // Render +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler2{::streamfx::obs::gs::debug_color_capture, "Storage"}; +#endif + obs_source_process_filter_end(_self, obs_get_base_effect(OBS_EFFECT_DEFAULT), 1, 1); + + // Reset GPU state + gs_blend_state_pop(); + gs_matrix_pop(); + } else { + obs_source_skip_video_filter(_self); + return; + } + + _output_color = _input->get_texture(); + _output_alpha = _output_color; + } + + try { // Process the captured input with the provider. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_convert, "Process"}; +#endif + switch (_provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_process(_output_color, _output_alpha); + break; +#endif + } + } catch (...) { + obs_source_skip_video_filter(_self); + return; + } + + _dirty = false; + } + + { // Draw the result for the next filter to use. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_render, "Render"}; +#endif + if (_effect->has_parameter("InputA", ::streamfx::obs::gs::effect_parameter::type::Texture)) { + _effect->get_parameter("InputA").set_texture(_output_color); + } + if (_effect->has_parameter("InputB", ::streamfx::obs::gs::effect_parameter::type::Texture)) { + _effect->get_parameter("InputB").set_texture(_output_alpha); + } + if (_effect->has_parameter("Threshold", ::streamfx::obs::gs::effect_parameter::type::Float)) { + _effect->get_parameter("Threshold").set_float(.666667); + } + if (_effect->has_parameter("ThresholdRange", ::streamfx::obs::gs::effect_parameter::type::Float)) { + _effect->get_parameter("ThresholdRange").set_float(.333333); + } + while (gs_effect_loop(_effect->get_object(), "DrawAlphaThreshold")) { + gs_draw_sprite(nullptr, 0, _size.first, _size.second); + } + } +} + +struct switch_provider_data_t { + virtual_greenscreen_provider provider; +}; + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::switch_provider( + virtual_greenscreen_provider provider) +{ + std::unique_lock ul(_provider_lock); + + // Safeguard against calls made from unlocked memory. + if (provider == _provider) { + return; + } + + // This doesn't work correctly. + // - Need to allow multiple switches at once because OBS is weird. + // - Doesn't guarantee that the task is properly killed off. + + // Log information. + D_LOG_INFO("Instance '%s' is switching provider from '%s' to '%s'.", obs_source_get_name(_self), cstring(_provider), + cstring(provider)); + + // 1.If there is an existing task, attempt to cancel it. + if (_provider_task) { + streamfx::threadpool()->pop(_provider_task); + } + + // 2. Build data to pass into the task. + auto spd = std::make_shared(); + spd->provider = _provider; + _provider = provider; + + // 3. Then spawn a new task to switch provider. + _provider_task = streamfx::threadpool()->push( + std::bind(&virtual_greenscreen_instance::task_switch_provider, this, std::placeholders::_1), spd); +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::task_switch_provider( + util::threadpool_data_t data) +{ + std::shared_ptr spd = std::static_pointer_cast(data); + + // 1. Mark the provider as no longer ready. + _provider_ready = false; + + // 2. Lock the provider from being used. + std::unique_lock ul(_provider_lock); + + try { + // 3. Unload the previous provider. + switch (spd->provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_unload(); + break; +#endif + default: + break; + } + + // 4. Load the new provider. + switch (_provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + nvvfxgs_load(); + { + auto data = obs_source_get_settings(_self); + nvvfxgs_update(data); + obs_data_release(data); + } + break; +#endif + default: + break; + } + + // Log information. + D_LOG_INFO("Instance '%s' switched provider from '%s' to '%s'.", obs_source_get_name(_self), + cstring(spd->provider), cstring(_provider)); + + // 5. Set the new provider as valid. + _provider_ready = true; + } catch (std::exception const& ex) { + // Log information. + D_LOG_ERROR("Instance '%s' failed switching provider with error: %s", obs_source_get_name(_self), ex.what()); + } +} + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_load() +{ + _nvidia_fx = std::make_shared<::streamfx::nvidia::vfx::greenscreen>(); +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_unload() +{ + _nvidia_fx.reset(); +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_size() +{ + if (!_nvidia_fx) { + return; + } + + _nvidia_fx->size(_size); +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_process( + std::shared_ptr<::streamfx::obs::gs::texture>& color, std::shared_ptr<::streamfx::obs::gs::texture>& alpha) +{ + if (!_nvidia_fx) { + return; + } + + alpha = _nvidia_fx->process(_input->get_texture()); + color = _nvidia_fx->get_color(); +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_properties(obs_properties_t* props) +{ + obs_properties_t* grp = obs_properties_create(); + obs_properties_add_group(props, ST_KEY_NVIDIA_GREENSCREEN, D_TRANSLATE(ST_I18N_NVIDIA_GREENSCREEN), + OBS_GROUP_NORMAL, grp); + + { + auto p = + obs_properties_add_list(grp, ST_KEY_NVIDIA_GREENSCREEN_MODE, D_TRANSLATE(ST_I18N_NVIDIA_GREENSCREEN_MODE), + OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT); + obs_property_list_add_int(p, D_TRANSLATE(ST_I18N_NVIDIA_GREENSCREEN_MODE_PERFORMANCE), + static_cast(::streamfx::nvidia::vfx::greenscreen_mode::PERFORMANCE)); + obs_property_list_add_int(p, D_TRANSLATE(ST_I18N_NVIDIA_GREENSCREEN_MODE_QUALITY), + static_cast(::streamfx::nvidia::vfx::greenscreen_mode::QUALITY)); + } +} + +void streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance::nvvfxgs_update(obs_data_t* data) +{ + if (!_nvidia_fx) + return; + + _nvidia_fx->set_mode( + static_cast<::streamfx::nvidia::vfx::greenscreen_mode>(obs_data_get_int(data, ST_KEY_NVIDIA_GREENSCREEN_MODE))); +} + +#endif + +//------------------------------------------------------------------------------ +// Factory +//------------------------------------------------------------------------------ +virtual_greenscreen_factory::~virtual_greenscreen_factory() {} + +virtual_greenscreen_factory::virtual_greenscreen_factory() +{ + bool any_available = false; + + // 1. Try and load any configured providers. +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + try { + // Load CVImage and Video Effects SDK. + _nvcuda = ::streamfx::nvidia::cuda::obs::get(); + _nvcvi = ::streamfx::nvidia::cv::cv::get(); + _nvvfx = ::streamfx::nvidia::vfx::vfx::get(); + _nvidia_available = true; + any_available |= _nvidia_available; + } catch (const std::exception& ex) { + _nvidia_available = false; + _nvvfx.reset(); + _nvcvi.reset(); + _nvcuda.reset(); + D_LOG_WARNING("Failed to make NVIDIA Greenscreen available due to error: %s", ex.what()); + } catch (...) { + _nvidia_available = false; + _nvvfx.reset(); + _nvcvi.reset(); + _nvcuda.reset(); + D_LOG_WARNING("Failed to make NVIDIA Greenscreen available.", nullptr); + } +#endif + + // 2. Check if any of them managed to load at all. + if (!any_available) { + D_LOG_ERROR("All supported Virtual Greenscreen providers failed to initialize, disabling effect.", 0); + return; + } + + // 3. In any other case, register the filter! + _info.id = S_PREFIX "filter-virtual-greenscreen"; + _info.type = OBS_SOURCE_TYPE_FILTER; + _info.output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_CUSTOM_DRAW /*| OBS_SOURCE_SRGB*/; + + set_resolution_enabled(true); + finish_setup(); +} + +const char* virtual_greenscreen_factory::get_name() +{ + return D_TRANSLATE(ST_I18N); +} + +void virtual_greenscreen_factory::get_defaults2(obs_data_t* data) +{ + obs_data_set_default_int(data, ST_KEY_PROVIDER, static_cast(virtual_greenscreen_provider::AUTOMATIC)); + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + obs_data_set_default_int(data, ST_KEY_NVIDIA_GREENSCREEN_MODE, + static_cast(::streamfx::nvidia::vfx::greenscreen_mode::QUALITY)); +#endif +} + +static bool modified_provider(obs_properties_t* props, obs_property_t*, obs_data_t* settings) noexcept +try { + return true; +} catch (const std::exception& ex) { + DLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what()); + return false; +} catch (...) { + DLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__); + return false; +} + +obs_properties_t* virtual_greenscreen_factory::get_properties2(virtual_greenscreen_instance* data) +{ + obs_properties_t* pr = obs_properties_create(); + +#ifdef ENABLE_FRONTEND + { + obs_properties_add_button2(pr, S_MANUAL_OPEN, D_TRANSLATE(S_MANUAL_OPEN), + virtual_greenscreen_factory::on_manual_open, nullptr); + } +#endif + + if (data) { + data->properties(pr); + } + + { // Advanced Settings + auto grp = obs_properties_create(); + obs_properties_add_group(pr, S_ADVANCED, D_TRANSLATE(S_ADVANCED), OBS_GROUP_NORMAL, grp); + + { + auto p = obs_properties_add_list(grp, ST_KEY_PROVIDER, D_TRANSLATE(ST_I18N_PROVIDER), OBS_COMBO_TYPE_LIST, + OBS_COMBO_FORMAT_INT); + obs_property_set_modified_callback(p, modified_provider); + obs_property_list_add_int(p, D_TRANSLATE(S_STATE_AUTOMATIC), + static_cast(virtual_greenscreen_provider::AUTOMATIC)); + obs_property_list_add_int(p, D_TRANSLATE(ST_I18N_PROVIDER_NVIDIA_GREENSCREEN), + static_cast(virtual_greenscreen_provider::NVIDIA_GREENSCREEN)); + } + } + + return pr; +} + +#ifdef ENABLE_FRONTEND +bool virtual_greenscreen_factory::on_manual_open(obs_properties_t* props, obs_property_t* property, void* data) +try { + streamfx::open_url(HELP_URL); + return false; +} catch (const std::exception& ex) { + D_LOG_ERROR("Failed to open manual due to error: %s", ex.what()); + return false; +} catch (...) { + D_LOG_ERROR("Failed to open manual due to unknown error.", ""); + return false; +} +#endif + +bool streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory::is_provider_available( + virtual_greenscreen_provider provider) +{ + switch (provider) { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + case virtual_greenscreen_provider::NVIDIA_GREENSCREEN: + return _nvidia_available; +#endif + default: + return false; + } +} + +virtual_greenscreen_provider streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory::find_ideal_provider() +{ + for (auto v : provider_priority) { + if (virtual_greenscreen_factory::get()->is_provider_available(v)) { + return v; + break; + } + } + return virtual_greenscreen_provider::AUTOMATIC; +} + +std::shared_ptr _video_superresolution_factory_instance = nullptr; + +void virtual_greenscreen_factory::initialize() +try { + if (!_video_superresolution_factory_instance) + _video_superresolution_factory_instance = std::make_shared(); +} catch (const std::exception& ex) { + D_LOG_ERROR("Failed to initialize due to error: %s", ex.what()); +} catch (...) { + D_LOG_ERROR("Failed to initialize due to unknown error.", ""); +} + +void virtual_greenscreen_factory::finalize() +{ + _video_superresolution_factory_instance.reset(); +} + +std::shared_ptr virtual_greenscreen_factory::get() +{ + return _video_superresolution_factory_instance; +} diff --git a/source/filters/filter-virtual-greenscreen.hpp b/source/filters/filter-virtual-greenscreen.hpp new file mode 100644 index 00000000..a2c5275f --- /dev/null +++ b/source/filters/filter-virtual-greenscreen.hpp @@ -0,0 +1,131 @@ +// Copyright (c) 2020 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include +#include +#include +#include "obs/gs/gs-effect.hpp" +#include "obs/gs/gs-rendertarget.hpp" +#include "obs/gs/gs-texture.hpp" +#include "obs/obs-source-factory.hpp" +#include "plugin.hpp" +#include "util/util-threadpool.hpp" + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA +#include "nvidia/vfx/nvidia-vfx-greenscreen.hpp" +#endif + +namespace streamfx::filter::virtual_greenscreen { + enum class virtual_greenscreen_provider { + INVALID = -1, + AUTOMATIC = 0, + NVIDIA_GREENSCREEN = 1, + }; + + const char* cstring(virtual_greenscreen_provider provider); + + std::string string(virtual_greenscreen_provider provider); + + class virtual_greenscreen_instance : public ::streamfx::obs::source_instance { + std::pair _size; + + std::atomic _provider; + virtual_greenscreen_provider _provider_ui; + std::atomic _provider_ready; + std::mutex _provider_lock; + std::shared_ptr _provider_task; + + std::shared_ptr<::streamfx::obs::gs::effect> _effect; + std::shared_ptr<::streamfx::obs::gs::sampler> _channel0_sampler; + std::shared_ptr<::streamfx::obs::gs::sampler> _channel1_sampler; + + std::shared_ptr<::streamfx::obs::gs::rendertarget> _input; + std::shared_ptr<::streamfx::obs::gs::texture> _output_color; + std::shared_ptr<::streamfx::obs::gs::texture> _output_alpha; + bool _dirty; + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + std::shared_ptr<::streamfx::nvidia::vfx::greenscreen> _nvidia_fx; +#endif + + public: + virtual_greenscreen_instance(obs_data_t* data, obs_source_t* self); + ~virtual_greenscreen_instance() override; + + void load(obs_data_t* data) override; + void migrate(obs_data_t* data, uint64_t version) override; + void update(obs_data_t* data) override; + void properties(obs_properties_t* properties); + + uint32_t get_width() override; + uint32_t get_height() override; + + void video_tick(float_t time) override; + void video_render(gs_effect_t* effect) override; + + private: + void switch_provider(virtual_greenscreen_provider provider); + void task_switch_provider(util::threadpool_data_t data); + +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + void nvvfxgs_load(); + void nvvfxgs_unload(); + void nvvfxgs_size(); + void nvvfxgs_process(std::shared_ptr<::streamfx::obs::gs::texture>& color, + std::shared_ptr<::streamfx::obs::gs::texture>& alpha); + void nvvfxgs_properties(obs_properties_t* props); + void nvvfxgs_update(obs_data_t* data); +#endif + }; + + class virtual_greenscreen_factory : public ::streamfx::obs::source_factory< + ::streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory, + ::streamfx::filter::virtual_greenscreen::virtual_greenscreen_instance> { +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN_NVIDIA + bool _nvidia_available; + std::shared_ptr<::streamfx::nvidia::cuda::obs> _nvcuda; + std::shared_ptr<::streamfx::nvidia::cv::cv> _nvcvi; + std::shared_ptr<::streamfx::nvidia::vfx::vfx> _nvvfx; +#endif + + public: + virtual ~virtual_greenscreen_factory(); + virtual_greenscreen_factory(); + + virtual const char* get_name() override; + + virtual void get_defaults2(obs_data_t* data) override; + virtual obs_properties_t* get_properties2(virtual_greenscreen_instance* data) override; + +#ifdef ENABLE_FRONTEND + static bool on_manual_open(obs_properties_t* props, obs_property_t* property, void* data); +#endif + + bool is_provider_available(virtual_greenscreen_provider); + virtual_greenscreen_provider find_ideal_provider(); + + public: // Singleton + static void initialize(); + static void finalize(); + static std::shared_ptr<::streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory> get(); + }; + +} // namespace streamfx::filter::virtual_greenscreen diff --git a/source/plugin.cpp b/source/plugin.cpp index cfb7af17..a6d8289c 100644 --- a/source/plugin.cpp +++ b/source/plugin.cpp @@ -65,6 +65,9 @@ #ifdef ENABLE_FILTER_UPSCALING #include "filters/filter-upscaling.hpp" #endif +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN +#include "filters/filter-virtual-greenscreen.hpp" +#endif #ifdef ENABLE_SOURCE_MIRROR #include "sources/source-mirror.hpp" @@ -175,6 +178,9 @@ try { #endif #ifdef ENABLE_FILTER_UPSCALING streamfx::filter::upscaling::upscaling_factory::initialize(); +#endif +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN + streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory::initialize(); #endif } @@ -267,6 +273,9 @@ try { #endif #ifdef ENABLE_FILTER_UPSCALING streamfx::filter::upscaling::upscaling_factory::finalize(); +#endif +#ifdef ENABLE_FILTER_VIRTUAL_GREENSCREEN + streamfx::filter::virtual_greenscreen::virtual_greenscreen_factory::finalize(); #endif }