From e6ec0fc4c7bac009cf8d4b4feebd90d262541f03 Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Mon, 25 Oct 2021 20:36:40 +0200 Subject: [PATCH] nvidia/ar/facedetection: Add Face Detection feature --- CMakeLists.txt | 2 + source/nvidia/ar/nvidia-ar-facedetection.cpp | 237 +++++++++++++++++++ source/nvidia/ar/nvidia-ar-facedetection.hpp | 70 ++++++ 3 files changed, 309 insertions(+) create mode 100644 source/nvidia/ar/nvidia-ar-facedetection.cpp create mode 100644 source/nvidia/ar/nvidia-ar-facedetection.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 527b091b..3db4f5e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1063,6 +1063,8 @@ if(HAVE_NVIDIA_AR_SDK) "source/nvidia/ar/nvidia-ar.cpp" "source/nvidia/ar/nvidia-ar-feature.hpp" "source/nvidia/ar/nvidia-ar-feature.cpp" + "source/nvidia/ar/nvidia-ar-facedetection.hpp" + "source/nvidia/ar/nvidia-ar-facedetection.cpp" ) list(APPEND PROJECT_LIBRARIES NVIDIA::AR diff --git a/source/nvidia/ar/nvidia-ar-facedetection.cpp b/source/nvidia/ar/nvidia-ar-facedetection.cpp new file mode 100644 index 00000000..b5822dbf --- /dev/null +++ b/source/nvidia/ar/nvidia-ar-facedetection.cpp @@ -0,0 +1,237 @@ +/* + * Modern effects for a modern Streamer + * Copyright (C) 2020 Michael Fabian Dirks + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include "nvidia-ar-facedetection.hpp" +#include +#include +#include "obs/gs/gs-helper.hpp" +#include "util/util-logging.hpp" + +#ifdef _DEBUG +#define ST_PREFIX "<%s> " +#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#else +#define ST_PREFIX " " +#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) +#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) +#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) +#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) +#endif + +// At the current moment, both FaceDetection and FaceBoxDetection only support 8 faces. +#define ST_MAX_TRACKED_FACES 8 + +using namespace ::streamfx::nvidia; + +streamfx::nvidia::ar::facedetection::~facedetection() +{ + D_LOG_DEBUG("Finalizing... (Addr: 0x%" PRIuPTR ")", this); +} + +streamfx::nvidia::ar::facedetection::facedetection() + : feature(FEATURE_FACE_DETECTION), _input(), _source(), _tmp(), _rects(), _rects_confidence(), _bboxes(), + _dirty(true) +{ + D_LOG_DEBUG("Initializing... (Addr: 0x%" PRIuPTR ")", this); + + // Assign CUDA Stream object. + if (auto err = set(P_NVAR_CONFIG "CUDAStream", _nvcuda->get_stream()); err != cv::result::SUCCESS) { + throw cv::exception("CUDAStream", err); + } + + // Prepare initial memory + _rects.resize(ST_MAX_TRACKED_FACES); + _rects_confidence.resize(ST_MAX_TRACKED_FACES); + + // Set up initial configuration + set_tracking_limit(1); + + // Attempt to load the feature. + if (auto err = feature::load(); err != cv::result::SUCCESS) { + throw cv::exception("Load", err); + } +} + +std::pair ar::facedetection::tracking_limit_range() +{ + return {1, ST_MAX_TRACKED_FACES}; +} + +size_t ar::facedetection::tracking_limit() +{ + return _rects.size(); +} + +void ar::facedetection::set_tracking_limit(size_t v) +{ + // Ensure there is always at least one face being tracked. + v = std::max(v, 1); + + // Resize all data. + _rects.resize(v); + _rects_confidence.resize(v); + + // Update bounding boxes structure. + _bboxes.rects = _rects.data(); + _bboxes.maximum = v; + _bboxes.current = 0; + + // Update feature. + if (auto err = set_object(P_NVAR_OUTPUT "BoundingBoxes", reinterpret_cast(&_bboxes), sizeof(bounds_t)); + err != cv::result::SUCCESS) { + throw cv::exception("BoundingBoxes", err); + } + if (auto err = set(P_NVAR_OUTPUT "BoundingBoxesConfidence", _rects_confidence); err != cv::result::SUCCESS) { + throw cv::exception("BoundingBoxesConfidence", err); + } + if (auto err = set(P_NVAR_CONFIG "Temporal", (v == 1)); err != cv::result::SUCCESS) { + throw cv::exception("Temporal", err); + } + + // Mark effect dirty for reload. + _dirty = true; +} + +void ar::facedetection::process(std::shared_ptr<::streamfx::obs::gs::texture> in) +{ + // Enter Graphics and CUDA context. + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = _nvcuda->get_context()->enter(); + +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_magenta, "NvAR Face Detection"}; +#endif + + // Resize if the size or scale was changed. + resize(in->get_width(), in->get_height()); + + // Reload effect if dirty. + if (_dirty) { + load(); + } + + { // Copy parameter to input. +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_copy, "Copy In -> Input"}; +#endif + gs_copy_texture(_input->get_texture()->get_object(), in->get_object()); + } + + { // Convert Input to Source format +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_convert, "Copy Input -> Source"}; +#endif + if (auto res = _nvcv->NvCVImage_Transfer(_input->get_image(), _source->get_image(), 1.f, + _nvcuda->get_stream()->get(), _tmp->get_image()); + res != ::streamfx::nvidia::cv::result::SUCCESS) { + D_LOG_ERROR("Failed to transfer input to processing source due to error: %s", + _nvcv->NvCV_GetErrorStringFromCode(res)); + throw std::runtime_error("Transfer failed."); + } + } + + { // Run +#ifdef ENABLE_PROFILING + ::streamfx::obs::gs::debug_marker profiler1{::streamfx::obs::gs::debug_color_cache, "Run"}; +#endif + if (auto err = run(); err != cv::result::SUCCESS) { + throw cv::exception("Run", err); + } + } +} + +size_t streamfx::nvidia::ar::facedetection::count() +{ + return _bboxes.current; +} + +streamfx::nvidia::ar::rect_t const& streamfx::nvidia::ar::facedetection::at(size_t index) +{ + float v; + return at(index, v); +} + +streamfx::nvidia::ar::rect_t const& streamfx::nvidia::ar::facedetection::at(size_t index, float& confidence) +{ + if (_bboxes.current == 0) + throw std::runtime_error("no tracked faces"); + if (index > _bboxes.current) + throw std::out_of_range("index too large"); + auto& ref = _rects.at(index); + confidence = _rects_confidence.at(index); + return ref; +} + +void ar::facedetection::resize(uint32_t width, uint32_t height) +{ + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + if (!_tmp) { + _tmp = std::make_shared<::streamfx::nvidia::cv::image>( + width, height, ::streamfx::nvidia::cv::pixel_format::RGBA, ::streamfx::nvidia::cv::component_type::UINT8, + ::streamfx::nvidia::cv::component_layout::PLANAR, ::streamfx::nvidia::cv::memory_location::GPU, 1); + } + + if (!_input || (width != _input->get_texture()->get_width()) || (height != _input->get_texture()->get_height())) { + if (_input) { + _input->resize(width, height); + } else { + _input = std::make_shared<::streamfx::nvidia::cv::texture>(width, height, GS_RGBA_UNORM); + } + _dirty = true; + } + + if (!_source || (width != _source->get_image()->width) || (height != _source->get_image()->height)) { + if (_source) { + _source->resize(width, height); + } else { + _source = std::make_shared<::streamfx::nvidia::cv::image>( + width, height, ::streamfx::nvidia::cv::pixel_format::BGR, ::streamfx::nvidia::cv::component_type::UINT8, + ::streamfx::nvidia::cv::component_layout::INTERLEAVED, ::streamfx::nvidia::cv::memory_location::GPU, 1); + } + + if (auto err = set(P_NVAR_INPUT "Image", _source); err != cv::result::SUCCESS) { + throw cv::exception("Image", err); + } + + _dirty = true; + } +} + +void streamfx::nvidia::ar::facedetection::load() +{ + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + // Assign CUDA Stream object. + if (auto err = set(P_NVAR_CONFIG "CUDAStream", _nvcuda->get_stream()); err != cv::result::SUCCESS) { + throw cv::exception("CUDAStream", err); + } + + // Attempt to load the feature. + if (auto err = feature::load(); err != cv::result::SUCCESS) { + throw cv::exception("Load", err); + } + + _dirty = false; +} diff --git a/source/nvidia/ar/nvidia-ar-facedetection.hpp b/source/nvidia/ar/nvidia-ar-facedetection.hpp new file mode 100644 index 00000000..238263d9 --- /dev/null +++ b/source/nvidia/ar/nvidia-ar-facedetection.hpp @@ -0,0 +1,70 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include "nvidia-ar-feature.hpp" +#include "nvidia/cuda/nvidia-cuda-gs-texture.hpp" +#include "nvidia/cuda/nvidia-cuda-obs.hpp" +#include "nvidia/cuda/nvidia-cuda.hpp" +#include "nvidia/cv/nvidia-cv-image.hpp" +#include "nvidia/cv/nvidia-cv-texture.hpp" +#include "obs/gs/gs-texture.hpp" + +namespace streamfx::nvidia::ar { + class facedetection : public feature { + std::shared_ptr<::streamfx::nvidia::cv::texture> _input; + std::shared_ptr<::streamfx::nvidia::cv::image> _source; + std::shared_ptr<::streamfx::nvidia::cv::image> _tmp; + + std::vector _rects; + std::vector _rects_confidence; + bounds_t _bboxes; + + bool _dirty; + + public: + ~facedetection(); + + /** Create a new face detection feature. + * + * Must be in a graphics and CUDA context when calling. + */ + facedetection(); + + std::pair tracking_limit_range(); + + size_t tracking_limit(); + + void set_tracking_limit(size_t v); + + void process(std::shared_ptr<::streamfx::obs::gs::texture> in); + + size_t count(); + + rect_t const& at(size_t index); + + rect_t const& at(size_t index, float& confidence); + + private: + void resize(uint32_t width, uint32_t height); + + void load(); + }; +} // namespace streamfx::nvidia::ar