From 02c6156f2e0aa566d9730d6960c9a0103256e61f Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Sat, 1 May 2021 18:31:13 +0200 Subject: [PATCH] nvidia/vfx: Add wrapper for NVIDIA Video Effects SDK --- CMakeLists.txt | 29 ++++- source/nvidia/vfx/nvidia-vfx.cpp | 193 +++++++++++++++++++++++++++++++ source/nvidia/vfx/nvidia-vfx.hpp | 105 +++++++++++++++++ 3 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 source/nvidia/vfx/nvidia-vfx.cpp create mode 100644 source/nvidia/vfx/nvidia-vfx.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e8d346ab..16bd9384 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -822,6 +822,23 @@ if(REQUIRE_NVIDIA_ARSDK AND D_PLATFORM_WINDOWS) set(HAVE_NVIDIA_ARSDK ${NVAR_FOUND}) endif() +#- NVIDIA Video Effects SDK +set(HAVE_NVIDIA_VFX_SDK OFF) +if(REQUIRE_NVIDIA_VFX_SDK AND D_PLATFORM_WINDOWS) + if(EXISTS "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-vfx-sdk/version.h") + set(HAVE_NVIDIA_VFX_SDK ON) + endif() + + if(NOT TARGET NVIDIA::VFX) + add_library(NVIDIA::VFX IMPORTED INTERFACE) + target_include_directories(nvARProxy + INTERFACE + "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-vfx-sdk/nvvfx/include/" + "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-vfx-sdk/nvvfx/source/" + ) + endif() +endif() + #- NVIDIA CUDA (Windows) set(HAVE_NVIDIA_CUDA OFF) if(REQUIRE_NVIDIA_CUDA AND D_PLATFORM_WINDOWS) @@ -960,7 +977,7 @@ if(HAVE_NVIDIA_ARSDK) ) endif() -if(HAVE_NVIDIA_ARSDK) +if(HAVE_NVIDIA_ARSDK OR HAVE_NVIDIA_VFX_SDK) list(APPEND PROJECT_PRIVATE_SOURCE "source/nvidia/cv/nvidia-cv.hpp" "source/nvidia/cv/nvidia-cv.cpp" @@ -971,6 +988,16 @@ if(HAVE_NVIDIA_ARSDK) ) endif() +if(HAVE_NVIDIA_VFX_SDK) + list(APPEND PROJECT_PRIVATE_SOURCE + "source/nvidia/vfx/nvidia-vfx.hpp" + "source/nvidia/vfx/nvidia-vfx.cpp" + ) + list(APPEND PROJECT_LIBRARIES + NVIDIA::VFX + ) +endif() + if(HAVE_NVIDIA_CUDA) list(APPEND PROJECT_PRIVATE_SOURCE "source/nvidia/cuda/nvidia-cuda.hpp" diff --git a/source/nvidia/vfx/nvidia-vfx.cpp b/source/nvidia/vfx/nvidia-vfx.cpp new file mode 100644 index 00000000..0367e9a9 --- /dev/null +++ b/source/nvidia/vfx/nvidia-vfx.cpp @@ -0,0 +1,193 @@ +// Copyright (c) 2020 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "nvidia-vfx.hpp" +#include +#include +#include "nvidia/cuda/nvidia-cuda-obs.hpp" +#include "obs/gs/gs-helper.hpp" +#include "util/util-logging.hpp" +#include "util/util-platform.hpp" + +#ifdef _DEBUG +#define ST_PREFIX "<%s> " +#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#else +#define ST_PREFIX " " +#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) +#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) +#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) +#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) +#endif + +#if defined(WIN32) +#include +#include +#include + +#define LIB_NAME "NVVideoEffects.dll" +#else +#define LIB_NAME "libNVVideoEffects.so" +#endif + +#define ST_ENV_NVIDIA_VIDEO_EFFECTS_SDK_PATH L"NV_VIDEO_EFFECTS_PATH" + +#define NVVFX_LOAD_SYMBOL(NAME) \ + { \ + NAME = reinterpret_cast(_library->load_symbol(#NAME)); \ + if (!NAME) \ + throw std::runtime_error("Failed to load '" #NAME "' from '" LIB_NAME "'."); \ + } + +streamfx::nvidia::vfx::vfx::~vfx() +{ + D_LOG_DEBUG("Finalizing... (Addr: 0x%" PRIuPTR ")", this); + + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + _library.reset(); +} + +streamfx::nvidia::vfx::vfx::vfx() +{ + std::filesystem::path sdk_path; + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + D_LOG_DEBUG("Initializing... (Addr: 0x%" PRIuPTR ")", this); + + // Figure out the location of the Video Effects SDK, if it is installed. +#ifdef WIN32 + { + DWORD env_size; + std::vector buffer; + + env_size = GetEnvironmentVariableW(ST_ENV_NVIDIA_VIDEO_EFFECTS_SDK_PATH, nullptr, 0); + if (env_size > 0) { + buffer.resize(static_cast(env_size) + 1); + env_size = GetEnvironmentVariableW(ST_ENV_NVIDIA_VIDEO_EFFECTS_SDK_PATH, buffer.data(), buffer.size()); + sdk_path = std::wstring(buffer.data(), buffer.size()); + } else { + PWSTR str = nullptr; + HRESULT res = SHGetKnownFolderPath(FOLDERID_ProgramFiles, KF_FLAG_DEFAULT, nullptr, &str); + if (res == S_OK) { + sdk_path = std::wstring(str); + CoTaskMemFree(str); + sdk_path /= "NVIDIA Corporation"; + sdk_path /= "NVIDIA Video Effects"; + } + } + } +#else + throw std::runtime_error("Not yet implemented."); +#endif + + // Check if any of the found paths are valid. + if (!std::filesystem::exists(sdk_path)) { + D_LOG_ERROR("No supported NVIDIA SDK is installed to provide '%s'.", LIB_NAME); + throw std::runtime_error("Failed to load '" LIB_NAME "'."); + } + +#ifdef WIN32 + // On platforms where it is possible, modify the linker directories. + SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + DLL_DIRECTORY_COOKIE ck = AddDllDirectory(sdk_path.wstring().c_str()); +#endif + + // Try and load the libraries + if (!_library) { + // Load it by name. + try { + _library = ::streamfx::util::library::load(std::string_view(LIB_NAME)); + } catch (...) { + // Load it by path. + auto lib_path = sdk_path; + lib_path /= LIB_NAME; + try { + _library = ::streamfx::util::library::load(util::platform::native_to_utf8(lib_path)); + } catch (std::exception const& ex) { + D_LOG_ERROR("Failed to load '%s' from '%s' with error: %s", LIB_NAME, + util::platform::native_to_utf8(lib_path).string().c_str(), ex.what()); + throw std::runtime_error("Failed to load '" LIB_NAME "'."); + } catch (...) { + D_LOG_ERROR("Failed to load '%s' from '%s'.", LIB_NAME, + util::platform::native_to_utf8(lib_path).string().c_str()); + throw std::runtime_error("Failed to load '" LIB_NAME "'."); + } + } + } + + // Store the model path for later use. + _model_path = std::filesystem::path(sdk_path) / "models"; + + { // Load Symbols + NVVFX_LOAD_SYMBOL(NvVFX_GetVersion); + NVVFX_LOAD_SYMBOL(NvVFX_CreateEffect); + NVVFX_LOAD_SYMBOL(NvVFX_DestroyEffect); + NVVFX_LOAD_SYMBOL(NvVFX_SetU32); + NVVFX_LOAD_SYMBOL(NvVFX_SetS32); + NVVFX_LOAD_SYMBOL(NvVFX_SetF32); + NVVFX_LOAD_SYMBOL(NvVFX_SetF64); + NVVFX_LOAD_SYMBOL(NvVFX_SetU64); + NVVFX_LOAD_SYMBOL(NvVFX_SetImage); + NVVFX_LOAD_SYMBOL(NvVFX_SetObject); + NVVFX_LOAD_SYMBOL(NvVFX_SetString); + NVVFX_LOAD_SYMBOL(NvVFX_SetCudaStream); + NVVFX_LOAD_SYMBOL(NvVFX_GetU32); + NVVFX_LOAD_SYMBOL(NvVFX_GetS32); + NVVFX_LOAD_SYMBOL(NvVFX_GetF32); + NVVFX_LOAD_SYMBOL(NvVFX_GetF64); + NVVFX_LOAD_SYMBOL(NvVFX_GetU64); + NVVFX_LOAD_SYMBOL(NvVFX_GetImage); + NVVFX_LOAD_SYMBOL(NvVFX_GetObject); + NVVFX_LOAD_SYMBOL(NvVFX_GetString); + NVVFX_LOAD_SYMBOL(NvVFX_GetCudaStream); + NVVFX_LOAD_SYMBOL(NvVFX_Run); + NVVFX_LOAD_SYMBOL(NvVFX_Load); + } + + { // Assign proper GPU. + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + NvVFX_SetU32(nullptr, PARAMETER_GPU, 0); + } +} + +std::shared_ptr<::streamfx::nvidia::vfx::vfx> streamfx::nvidia::vfx::vfx::get() +{ + static std::weak_ptr instance; + static std::mutex lock; + + std::unique_lock ul(lock); + if (instance.expired()) { + auto hard_instance = std::make_shared(); + instance = hard_instance; + return hard_instance; + } + return instance.lock(); +} + +std::filesystem::path streamfx::nvidia::vfx::vfx::model_path() +{ + return _model_path; +} diff --git a/source/nvidia/vfx/nvidia-vfx.hpp b/source/nvidia/vfx/nvidia-vfx.hpp new file mode 100644 index 00000000..e7e8488b --- /dev/null +++ b/source/nvidia/vfx/nvidia-vfx.hpp @@ -0,0 +1,105 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include +#include "nvidia/cv/nvidia-cv.hpp" + +#define NVVFX_DEFINE_FUNCTION(name, ...) \ + private: \ + typedef ::streamfx::nvidia::cv::result(__cdecl* t##name)(__VA_ARGS__); \ + \ + public: \ + t##name name = nullptr; + +namespace streamfx::nvidia::vfx { + typedef const char* effect_t; + typedef const char* parameter_t; + typedef void* object_t; + typedef object_t handle_t; + + static constexpr effect_t EFFECT_TRANSFER = "Transfer"; + static constexpr effect_t EFFECT_GREEN_SCREEN = "GreenScreen"; + static constexpr effect_t EFFECT_BACKGROUND_BLUR = "BackgroundBlur"; + static constexpr effect_t EFFECT_ARTIFACT_REDUCTION = "ArtifactReduction"; + static constexpr effect_t EFFECT_SUPERRESOLUTION = "SuperRes"; + static constexpr effect_t EFFECT_UPSCALE = "Upscale"; + static constexpr effect_t EFFECT_DENOISING = "Denoising"; + + static constexpr parameter_t PARAMETER_INPUT_IMAGE_0 = "SrcImage0"; + static constexpr parameter_t PARAMETER_INPUT_IMAGE_1 = "SrcImage1"; + static constexpr parameter_t PARAMETER_OUTPUT_IMAGE_0 = "DstImage0"; + static constexpr parameter_t PARAMETER_MODEL_DIRECTORY = "ModelDir"; + static constexpr parameter_t PARAMETER_CUDA_STREAM = "CudaStream"; + static constexpr parameter_t PARAMETER_INFO = "Info"; + static constexpr parameter_t PARAMETER_SCALE = "Scale"; + static constexpr parameter_t PARAMETER_STRENGTH = "Strength"; + static constexpr parameter_t PARAMETER_STRENGTH_LEVELS = "StrengthLevels"; + static constexpr parameter_t PARAMETER_MODE = "Mode"; + static constexpr parameter_t PARAMETER_TEMPORAL = "Temporal"; + static constexpr parameter_t PARAMETER_GPU = "GPU"; + static constexpr parameter_t PARAMETER_BATCH_SIZE = "BatchSize"; + static constexpr parameter_t PARAMETER_MODEL_BATCH = "ModelBatch"; + static constexpr parameter_t PARAMETER_STATE = "State"; + static constexpr parameter_t PARAMETER_STATE_SIZE = "StateSize"; + + class vfx { + std::shared_ptr<::streamfx::util::library> _library; + std::filesystem::path _model_path; + + public: + ~vfx(); + vfx(); + + std::filesystem::path model_path(); + + public: + NVVFX_DEFINE_FUNCTION(NvVFX_GetVersion, uint32_t* version); + NVVFX_DEFINE_FUNCTION(NvVFX_CreateEffect, effect_t effect, handle_t* handle); + NVVFX_DEFINE_FUNCTION(NvVFX_DestroyEffect, handle_t handle); + NVVFX_DEFINE_FUNCTION(NvVFX_SetU32, handle_t effect, parameter_t paramName, uint32_t val); + NVVFX_DEFINE_FUNCTION(NvVFX_SetS32, handle_t effect, parameter_t paramName, int32_t val); + NVVFX_DEFINE_FUNCTION(NvVFX_SetF32, handle_t effect, parameter_t paramName, float val); + NVVFX_DEFINE_FUNCTION(NvVFX_SetF64, handle_t effect, parameter_t paramName, double val); + NVVFX_DEFINE_FUNCTION(NvVFX_SetU64, handle_t effect, parameter_t paramName, uint64_t val); + NVVFX_DEFINE_FUNCTION(NvVFX_SetObject, handle_t effect, parameter_t paramName, void* ptr); + NVVFX_DEFINE_FUNCTION(NvVFX_SetCudaStream, handle_t effect, parameter_t paramName, + ::streamfx::nvidia::cuda::stream_t stream); + NVVFX_DEFINE_FUNCTION(NvVFX_SetImage, handle_t effect, parameter_t paramName, + ::streamfx::nvidia::cv::image_t* im); + NVVFX_DEFINE_FUNCTION(NvVFX_SetString, handle_t effect, parameter_t paramName, const char* str); + NVVFX_DEFINE_FUNCTION(NvVFX_GetU32, handle_t effect, parameter_t paramName, uint32_t* val); + NVVFX_DEFINE_FUNCTION(NvVFX_GetS32, handle_t effect, parameter_t paramName, int32_t* val); + NVVFX_DEFINE_FUNCTION(NvVFX_GetF32, handle_t effect, parameter_t paramName, float* val); + NVVFX_DEFINE_FUNCTION(NvVFX_GetF64, handle_t effect, parameter_t paramName, double* val); + NVVFX_DEFINE_FUNCTION(NvVFX_GetU64, handle_t effect, parameter_t paramName, uint64_t* val); + NVVFX_DEFINE_FUNCTION(NvVFX_GetObject, handle_t effect, parameter_t paramName, void** ptr); + NVVFX_DEFINE_FUNCTION(NvVFX_GetCudaStream, handle_t effect, parameter_t paramName, + ::streamfx::nvidia::cuda::stream_t stream); + NVVFX_DEFINE_FUNCTION(NvVFX_GetImage, handle_t effect, parameter_t paramName, + ::streamfx::nvidia::cv::image_t* im); + NVVFX_DEFINE_FUNCTION(NvVFX_GetString, handle_t effect, parameter_t paramName, const char** str); + NVVFX_DEFINE_FUNCTION(NvVFX_Run, handle_t effect, int32_t async); + NVVFX_DEFINE_FUNCTION(NvVFX_Load, handle_t effect); + + public: + static std::shared_ptr<::streamfx::nvidia::vfx::vfx> get(); + }; +} // namespace streamfx::nvidia::vfx