From 6d20280956bce1cdcffded7bdbebc9d56d423e92 Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Sat, 23 Oct 2021 17:49:06 +0200 Subject: [PATCH] nvidia/vfx/greenscreen: Simple wrapper for the Green Screen effect --- CMakeLists.txt | 2 + source/nvidia/vfx/nvidia-vfx-greenscreen.cpp | 293 +++++++++++++++++++ source/nvidia/vfx/nvidia-vfx-greenscreen.hpp | 65 ++++ 3 files changed, 360 insertions(+) create mode 100644 source/nvidia/vfx/nvidia-vfx-greenscreen.cpp create mode 100644 source/nvidia/vfx/nvidia-vfx-greenscreen.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f010af2d..e3055284 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1068,6 +1068,8 @@ if(HAVE_NVIDIA_VFX_SDK) "source/nvidia/vfx/nvidia-vfx-denoising.cpp" "source/nvidia/vfx/nvidia-vfx-effect.hpp" "source/nvidia/vfx/nvidia-vfx-effect.cpp" + "source/nvidia/vfx/nvidia-vfx-greenscreen.hpp" + "source/nvidia/vfx/nvidia-vfx-greenscreen.cpp" "source/nvidia/vfx/nvidia-vfx-superresolution.hpp" "source/nvidia/vfx/nvidia-vfx-superresolution.cpp" ) diff --git a/source/nvidia/vfx/nvidia-vfx-greenscreen.cpp b/source/nvidia/vfx/nvidia-vfx-greenscreen.cpp new file mode 100644 index 00000000..a5df4fc3 --- /dev/null +++ b/source/nvidia/vfx/nvidia-vfx-greenscreen.cpp @@ -0,0 +1,293 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "nvidia-vfx-greenscreen.hpp" +#include +#include +#include "nvidia/cv/nvidia-cv.hpp" +#include "obs/gs/gs-helper.hpp" +#include "util/util-logging.hpp" +#include "util/utility.hpp" + +#ifdef _DEBUG +#define ST_PREFIX "<%s> " +#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#else +#define ST_PREFIX " " +#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) +#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) +#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) +#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) +#endif + +// TODO: Figure out actual latency, appears to be either 2 or 3 frames. +#define LATENCY_BUFFER 2 + +streamfx::nvidia::vfx::greenscreen::~greenscreen() +{ + // Enter Contexts. + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + _tmp.reset(); + _output.reset(); + _destination.reset(); + _source.reset(); + _input.reset(); + _buffer.clear(); +} + +streamfx::nvidia::vfx::greenscreen::greenscreen() + : effect(EFFECT_GREEN_SCREEN), _dirty(true), _input(), _source(), _destination(), _output(), _tmp() +{ + // Enter Contexts. + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + // Assign CUDA Stream object. + if (auto v = set(PARAMETER_CUDA_STREAM, _nvcuda->get_stream()); v != cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception(PARAMETER_CUDA_STREAM, v); + } + + // Assign Model Directory. + if (auto v = set(PARAMETER_MODEL_DIRECTORY, _nvvfx->model_path().generic_u8string()); v != cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception(PARAMETER_MODEL_DIRECTORY, v); + } + + // Mode + set_mode(greenscreen_mode::QUALITY); + + // Allocate resources + resize(512, 288); +} + +void streamfx::nvidia::vfx::greenscreen::size(std::pair& size) +{ + constexpr uint32_t min_width = 512; + constexpr uint32_t min_height = 288; + + // Calculate Size + if (size.first > size.second) { + // Dominant Width + double ar = static_cast(size.second) / static_cast(size.first); + size.first = std::max(size.first, min_width); + size.second = + std::max(static_cast(std::lround(static_cast(size.first) * ar)), min_height); + } else { + // Dominant Height + double ar = static_cast(size.first) / static_cast(size.second); + size.second = std::max(size.second, min_height); + size.first = + std::max(static_cast(std::lround(static_cast(size.second) * ar)), min_width); + } +} + +void streamfx::nvidia::vfx::greenscreen::set_mode(greenscreen_mode mode) +{ + set(PARAMETER_MODE, static_cast(mode)); + _dirty = true; +} + +std::shared_ptr + streamfx::nvidia::vfx::greenscreen::process(std::shared_ptr<::streamfx::obs::gs::texture> in) +{ + // Enter Graphics and CUDA context. + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = _nvcuda->get_context()->enter(); + +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_magenta, "NvVFX Background Removal"}; +#endif + + // Resize if the size or scale was changed. + resize(in->get_width(), in->get_height()); + + // Reload effect if dirty. + if (_dirty) { + load(); + } + + { // Copy parameter to input. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_copy, "Copy In -> Input"}; +#endif + gs_copy_texture(_input->get_texture()->get_object(), in->get_object()); + } + + { // Enqueue into buffer (back is newest). + auto el = _buffer.front(); + gs_copy_texture(el->get_object(), in->get_object()); + _buffer.push_back(el); + _buffer.pop_front(); + } + + { // Copy input to source. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_copy, "Copy Input -> Source"}; +#endif + if (auto res = _nvcvi->NvCVImage_Transfer(_input->get_image(), _source->get_image(), 1.f, + _nvcuda->get_stream()->get(), _tmp->get_image()); + res != ::streamfx::nvidia::cv::result::SUCCESS) { + D_LOG_ERROR("Failed to transfer input to processing source due to error: %s", + _nvcvi->NvCV_GetErrorStringFromCode(res)); + throw std::runtime_error("Transfer failed."); + } + } + + { // Process source to destination. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_cache, "Process"}; +#endif + if (auto res = run(); res != ::streamfx::nvidia::cv::result::SUCCESS) { + D_LOG_ERROR("Failed to process due to error: %s", _nvcvi->NvCV_GetErrorStringFromCode(res)); + throw std::runtime_error("Run failed."); + } + } + + { // Copy destination to output. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_copy, + "Copy Destination -> Output"}; +#endif + if (auto res = _nvcvi->NvCVImage_Transfer(_destination->get_image(), _output->get_image(), 1., + _nvcuda->get_stream()->get(), _tmp->get_image()); + res != ::streamfx::nvidia::cv::result::SUCCESS) { + D_LOG_ERROR("Failed to transfer processing result to output due to error: %s", + _nvcvi->NvCV_GetErrorStringFromCode(res)); + throw std::runtime_error("Transfer failed."); + } + } + + // Return output. + return _output->get_texture(); +} + +std::shared_ptr streamfx::nvidia::vfx::greenscreen::get_color() +{ + //return _input->get_texture(); + return _buffer.front(); +} + +std::shared_ptr streamfx::nvidia::vfx::greenscreen::get_mask() +{ + return _output->get_texture(); +} + +void streamfx::nvidia::vfx::greenscreen::resize(uint32_t width, uint32_t height) +{ + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + std::pair in_size = {width, height}; + size(in_size); + + if (!_tmp) { + _tmp = std::make_shared<::streamfx::nvidia::cv::image>( + width, height, ::streamfx::nvidia::cv::pixel_format::RGBA, ::streamfx::nvidia::cv::component_type::UINT8, + ::streamfx::nvidia::cv::component_layout::PLANAR, ::streamfx::nvidia::cv::memory_location::GPU, 1); + } + + if (!_input || (in_size.first != _input->get_texture()->get_width()) + || (in_size.second != _input->get_texture()->get_height())) { + { + _buffer.clear(); + for (size_t idx = 0; idx < LATENCY_BUFFER; idx++) { + auto el = std::make_shared<::streamfx::obs::gs::texture>(width, height, GS_RGBA_UNORM, 1, nullptr, + ::streamfx::obs::gs::texture::flags::None); + _buffer.push_back(el); + } + } + + if (_input) { + _input->resize(in_size.first, in_size.second); + } else { + _input = std::make_shared<::streamfx::nvidia::cv::texture>(in_size.first, in_size.second, GS_RGBA_UNORM); + } + + _dirty = true; + } + + if (!_source || (in_size.first != _source->get_image()->width) + || (in_size.second != _source->get_image()->height)) { + if (_source) { + _source->resize(in_size.first, in_size.second); + } else { + _source = std::make_shared<::streamfx::nvidia::cv::image>( + in_size.first, in_size.second, ::streamfx::nvidia::cv::pixel_format::BGR, + ::streamfx::nvidia::cv::component_type::UINT8, ::streamfx::nvidia::cv::component_layout::INTERLEAVED, + ::streamfx::nvidia::cv::memory_location::GPU, 1); + } + + if (auto v = set(PARAMETER_INPUT_IMAGE_0, _source); v != ::streamfx::nvidia::cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception(PARAMETER_INPUT_IMAGE_0, v); + } + + _dirty = true; + } + + if (!_destination || (in_size.first != _destination->get_image()->width) + || (in_size.second != _destination->get_image()->height)) { + if (_destination) { + _destination->resize(in_size.first, in_size.second); + } else { + _destination = std::make_shared<::streamfx::nvidia::cv::image>( + in_size.first, in_size.second, ::streamfx::nvidia::cv::pixel_format::A, + ::streamfx::nvidia::cv::component_type::UINT8, ::streamfx::nvidia::cv::component_layout::INTERLEAVED, + ::streamfx::nvidia::cv::memory_location::GPU, 1); + } + + if (auto v = set(PARAMETER_OUTPUT_IMAGE_0, _destination); v != ::streamfx::nvidia::cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception(PARAMETER_OUTPUT_IMAGE_0, v); + } + + _dirty = true; + } + + if (!_output || (in_size.first != _output->get_texture()->get_width()) + || (in_size.second != _output->get_texture()->get_height())) { + if (_output) { + _output->resize(in_size.first, in_size.second); + } else { + _output = std::make_shared<::streamfx::nvidia::cv::texture>(in_size.first, in_size.second, GS_A8); + } + + _dirty = true; + } +} + +void streamfx::nvidia::vfx::greenscreen::load() +{ + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = _nvcuda->get_context()->enter(); + + // Assign CUDA Stream object. + if (auto v = set(PARAMETER_CUDA_STREAM, _nvcuda->get_stream()); v != cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception(PARAMETER_CUDA_STREAM, v); + } + + if (auto v = effect::load(); v != ::streamfx::nvidia::cv::result::SUCCESS) { + throw ::streamfx::nvidia::cv::exception("load", v); + } + + _dirty = false; +} diff --git a/source/nvidia/vfx/nvidia-vfx-greenscreen.hpp b/source/nvidia/vfx/nvidia-vfx-greenscreen.hpp new file mode 100644 index 00000000..a53f7402 --- /dev/null +++ b/source/nvidia/vfx/nvidia-vfx-greenscreen.hpp @@ -0,0 +1,65 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include "nvidia-vfx-effect.hpp" +#include "nvidia-vfx.hpp" +#include "nvidia/cuda/nvidia-cuda-gs-texture.hpp" +#include "nvidia/cuda/nvidia-cuda-obs.hpp" +#include "nvidia/cuda/nvidia-cuda.hpp" +#include "nvidia/cv/nvidia-cv-image.hpp" +#include "nvidia/cv/nvidia-cv-texture.hpp" +#include "obs/gs/gs-texture.hpp" + +namespace streamfx::nvidia::vfx { + enum class greenscreen_mode { + QUALITY = 0, + PERFORMANCE = 1, + }; + + class greenscreen : protected effect { + bool _dirty; + std::list> _buffer; + std::shared_ptr<::streamfx::nvidia::cv::texture> _input; + std::shared_ptr<::streamfx::nvidia::cv::image> _source; + std::shared_ptr<::streamfx::nvidia::cv::image> _destination; + std::shared_ptr<::streamfx::nvidia::cv::texture> _output; + std::shared_ptr<::streamfx::nvidia::cv::image> _tmp; + + public: + ~greenscreen(); + greenscreen(); + + void size(std::pair& size); + + void set_mode(greenscreen_mode mode); + + std::shared_ptr<::streamfx::obs::gs::texture> process(std::shared_ptr<::streamfx::obs::gs::texture> in); + + std::shared_ptr<::streamfx::obs::gs::texture> get_color(); + + std::shared_ptr<::streamfx::obs::gs::texture> get_mask(); + + private: + void resize(uint32_t width, uint32_t height); + + void load(); + }; +} // namespace streamfx::nvidia::vfx