mirror of
https://github.com/Xaymar/obs-StreamFX
synced 2024-11-13 07:15:06 +00:00
filter/auto-framing: Frame your face, automatically!
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
Xaymar
parent
368b366219
commit
323a6dcc9f
5 changed files with 1580 additions and 0 deletions
|
|
@ -297,6 +297,8 @@ set(${PREFIX}ENABLE_ENCODER_FFMPEG_PRORES ON CACHE BOOL "Enable ProRes Encoder i
|
|||
set(${PREFIX}ENABLE_ENCODER_AOM_AV1 ON CACHE BOOL "Enable AOM AV1 Encoder.")
|
||||
|
||||
## Filters
|
||||
set(${PREFIX}ENABLE_FILTER_AUTOFRAMING ON CACHE BOOL "Enable Auto-Framing Filter")
|
||||
set(${PREFIX}ENABLE_FILTER_AUTOFRAMING_NVIDIA ON CACHE BOOL "Enable NVIDIA provider(s) Auto-Framing Filter")
|
||||
set(${PREFIX}ENABLE_FILTER_BLUR ON CACHE BOOL "Enable Blur Filter")
|
||||
set(${PREFIX}ENABLE_FILTER_COLOR_GRADE ON CACHE BOOL "Enable Color Grade Filter")
|
||||
set(${PREFIX}ENABLE_FILTER_DENOISING ON CACHE BOOL "Enable Denoising filter")
|
||||
|
|
@ -652,6 +654,27 @@ function(feature_encoder_aom_av1 RESOLVE)
|
|||
endif()
|
||||
endfunction()
|
||||
|
||||
function(feature_filter_autoframing RESOLVE)
|
||||
is_feature_enabled(FILTER_AUTOFRAMING T_CHECK)
|
||||
if(RESOLVE AND T_CHECK)
|
||||
# Verify that the requirements for the providers are available
|
||||
if(NOT HAVE_NVIDIA_AR_SDK)
|
||||
message(WARNING "${LOGPREFIX}: 'NVIDIA Augmented Reality SDK' is missing. Disabling NVIDIA provider...")
|
||||
set_feature_disabled(FILTER_AUTOFRAMING_NVIDIA ON)
|
||||
endif()
|
||||
|
||||
# Verify that we have at least one provider for Auto-Framing.
|
||||
is_feature_enabled(FILTER_AUTOFRAMING_NVIDIA T_CHECK_NVIDIA)
|
||||
if (NOT T_CHECK_NVIDIA)
|
||||
message(WARNING "${LOGPREFIX}: Auto-Framing has no available providers. Disabling...")
|
||||
set_feature_disabled(FILTER_AUTOFRAMING ON)
|
||||
endif()
|
||||
elseif(T_CHECK)
|
||||
set(REQUIRE_NVIDIA_AR_SDK ON PARENT_SCOPE)
|
||||
set(REQUIRE_NVIDIA_CUDA ON PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(feature_filter_blur RESOLVE)
|
||||
is_feature_enabled(FILTER_BLUR T_CHECK)
|
||||
endfunction()
|
||||
|
|
@ -787,6 +810,7 @@ endfunction()
|
|||
# Set Requirements
|
||||
feature_encoder_ffmpeg(OFF)
|
||||
feature_encoder_aom_av1(OFF)
|
||||
feature_filter_autoframing(OFF)
|
||||
feature_filter_blur(OFF)
|
||||
feature_filter_color_grade(OFF)
|
||||
feature_filter_denoising(OFF)
|
||||
|
|
@ -950,6 +974,7 @@ endif()
|
|||
# Verify Requirements
|
||||
feature_encoder_ffmpeg(ON)
|
||||
feature_encoder_aom_av1(ON)
|
||||
feature_filter_autoframing(ON)
|
||||
feature_filter_blur(ON)
|
||||
feature_filter_color_grade(ON)
|
||||
feature_filter_denoising(ON)
|
||||
|
|
@ -1300,6 +1325,24 @@ if(T_CHECK)
|
|||
)
|
||||
endif()
|
||||
|
||||
# Filter/Auto-Framing
|
||||
is_feature_enabled(FILTER_AUTOFRAMING T_CHECK)
|
||||
if(T_CHECK)
|
||||
list(APPEND PROJECT_PRIVATE_SOURCE
|
||||
"source/filters/filter-autoframing.hpp"
|
||||
"source/filters/filter-autoframing.cpp"
|
||||
)
|
||||
list(APPEND PROJECT_DEFINITIONS
|
||||
ENABLE_FILTER_AUTOFRAMING
|
||||
)
|
||||
is_feature_enabled(FILTER_AUTOFRAMING_NVIDIA T_CHECK)
|
||||
if (T_CHECK)
|
||||
list(APPEND PROJECT_DEFINITIONS
|
||||
ENABLE_FILTER_AUTOFRAMING_NVIDIA
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Filter/Blur
|
||||
is_feature_enabled(FILTER_BLUR T_CHECK)
|
||||
if(T_CHECK)
|
||||
|
|
|
|||
|
|
@ -242,6 +242,24 @@ Filter.Shader="Shader"
|
|||
Source.Shader="Shader"
|
||||
Transition.Shader="Shader"
|
||||
|
||||
# Filter - Auto-Framing
|
||||
Filter.AutoFraming="Auto-Framing"
|
||||
Filter.AutoFraming.Tracking="Tracking Options"
|
||||
Filter.AutoFraming.Tracking.Mode="Mode"
|
||||
Filter.AutoFraming.Tracking.Mode.Solo="Solo"
|
||||
Filter.AutoFraming.Tracking.Mode.Group="Group"
|
||||
Filter.AutoFraming.Tracking.Frequency="Frequency"
|
||||
Filter.AutoFraming.Motion="Motion Options"
|
||||
Filter.AutoFraming.Motion.Smoothing="Smoothing"
|
||||
Filter.AutoFraming.Motion.Prediction="Prediction"
|
||||
Filter.AutoFraming.Framing="Framing Options"
|
||||
Filter.AutoFraming.Framing.Stability="Stability"
|
||||
Filter.AutoFraming.Framing.Padding="Padding"
|
||||
Filter.AutoFraming.Framing.Offset="Offset"
|
||||
Filter.AutoFraming.Framing.AspectRatio="Aspect Ratio"
|
||||
Filter.AutoFraming.Provider="Provider"
|
||||
Filter.AutoFraming.Provider.NVIDIA.FaceDetection="NVIDIA® Face Detection, powered by NVIDIA® Broadcast"
|
||||
|
||||
# Filter - Blur
|
||||
Filter.Blur="Blur"
|
||||
Filter.Blur.Type="Type"
|
||||
|
|
|
|||
1321
source/filters/filter-autoframing.cpp
Normal file
1321
source/filters/filter-autoframing.cpp
Normal file
File diff suppressed because it is too large
Load diff
189
source/filters/filter-autoframing.hpp
Normal file
189
source/filters/filter-autoframing.hpp
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
/*
|
||||
* Modern effects for a modern Streamer
|
||||
* Copyright (C) 2020 Michael Fabian Dirks
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include "gfx/gfx-debug.hpp"
|
||||
#include "obs/gs/gs-rendertarget.hpp"
|
||||
#include "obs/gs/gs-texture.hpp"
|
||||
#include "obs/gs/gs-vertexbuffer.hpp"
|
||||
#include "obs/obs-source-factory.hpp"
|
||||
#include "plugin.hpp"
|
||||
#include "util/util-threadpool.hpp"
|
||||
#include "util/utility.hpp"
|
||||
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING_NVIDIA
|
||||
#include "nvidia/ar/nvidia-ar-facedetection.hpp"
|
||||
#endif
|
||||
|
||||
namespace streamfx::filter::autoframing {
|
||||
|
||||
enum class tracking_mode : int64_t {
|
||||
SOLO = 0,
|
||||
GROUP = 1,
|
||||
};
|
||||
|
||||
enum class tracking_provider : int64_t {
|
||||
INVALID = -1,
|
||||
AUTOMATIC = 0,
|
||||
NVIDIA_FACEDETECTION = 1,
|
||||
};
|
||||
|
||||
const char* cstring(tracking_provider provider);
|
||||
|
||||
std::string string(tracking_provider provider);
|
||||
|
||||
class autoframing_instance : public obs::source_instance {
|
||||
struct track_el {
|
||||
float age;
|
||||
vec2 pos;
|
||||
vec2 size;
|
||||
vec2 vel;
|
||||
};
|
||||
|
||||
struct pred_el {
|
||||
// Motion-Predicted Position
|
||||
vec2 mp_pos;
|
||||
|
||||
// Filtered Position
|
||||
streamfx::util::math::kalman1D<float> filter_pos_x;
|
||||
streamfx::util::math::kalman1D<float> filter_pos_y;
|
||||
|
||||
// Offset Filtered Position
|
||||
vec2 offset_pos;
|
||||
|
||||
// Padded Area
|
||||
vec2 pad_size;
|
||||
|
||||
// Aspect-Ratio-Corrected Padded Area
|
||||
vec2 aspected_size;
|
||||
};
|
||||
|
||||
bool _dirty;
|
||||
std::pair<uint32_t, uint32_t> _size;
|
||||
std::pair<uint32_t, uint32_t> _out_size;
|
||||
|
||||
std::shared_ptr<::streamfx::gfx::debug> _gfx_debug;
|
||||
std::shared_ptr<::streamfx::obs::gs::effect> _standard_effect;
|
||||
std::shared_ptr<::streamfx::obs::gs::rendertarget> _input;
|
||||
std::shared_ptr<::streamfx::obs::gs::vertex_buffer> _vb;
|
||||
|
||||
tracking_provider _provider;
|
||||
tracking_provider _provider_ui;
|
||||
std::atomic<bool> _provider_ready;
|
||||
std::mutex _provider_lock;
|
||||
std::shared_ptr<util::threadpool::task> _provider_task;
|
||||
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING_NVIDIA
|
||||
std::shared_ptr<::streamfx::nvidia::ar::facedetection> _nvidia_fx;
|
||||
#endif
|
||||
|
||||
tracking_mode _track_mode;
|
||||
float _track_frequency;
|
||||
|
||||
float _motion_smoothing;
|
||||
float _motion_smoothing_kalman_pnc;
|
||||
float _motion_smoothing_kalman_mnc;
|
||||
float _motion_prediction;
|
||||
|
||||
float _frame_stability;
|
||||
float _frame_stability_kalman;
|
||||
bool _frame_padding_prc[2];
|
||||
vec2 _frame_padding;
|
||||
bool _frame_offset_prc[2];
|
||||
vec2 _frame_offset;
|
||||
float _frame_aspect_ratio;
|
||||
|
||||
float _track_frequency_counter;
|
||||
std::list<std::shared_ptr<track_el>> _tracked_elements;
|
||||
std::map<std::shared_ptr<track_el>, std::shared_ptr<pred_el>> _predicted_elements;
|
||||
|
||||
streamfx::util::math::kalman1D<float> _frame_pos_x;
|
||||
streamfx::util::math::kalman1D<float> _frame_pos_y;
|
||||
vec2 _frame_pos;
|
||||
streamfx::util::math::kalman1D<float> _frame_size_x;
|
||||
streamfx::util::math::kalman1D<float> _frame_size_y;
|
||||
vec2 _frame_size;
|
||||
|
||||
bool _debug;
|
||||
|
||||
public:
|
||||
~autoframing_instance();
|
||||
autoframing_instance(obs_data_t* settings, obs_source_t* self);
|
||||
|
||||
void load(obs_data_t* data) override;
|
||||
void migrate(obs_data_t* data, uint64_t version) override;
|
||||
void update(obs_data_t* data) override;
|
||||
void properties(obs_properties_t* properties);
|
||||
|
||||
uint32_t get_width() override;
|
||||
uint32_t get_height() override;
|
||||
|
||||
virtual void video_tick(float_t seconds) override;
|
||||
virtual void video_render(gs_effect_t* effect) override;
|
||||
|
||||
private:
|
||||
void tracking_tick(float seconds);
|
||||
|
||||
void switch_provider(tracking_provider provider);
|
||||
void task_switch_provider(util::threadpool_data_t data);
|
||||
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING_NVIDIA
|
||||
void nvar_facedetection_load();
|
||||
void nvar_facedetection_unload();
|
||||
void nvar_facedetection_process();
|
||||
void nvar_facedetection_properties(obs_properties_t* props);
|
||||
void nvar_facedetection_update(obs_data_t* data);
|
||||
#endif
|
||||
};
|
||||
|
||||
class autoframing_factory : public obs::source_factory<streamfx::filter::autoframing::autoframing_factory,
|
||||
streamfx::filter::autoframing::autoframing_instance> {
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING_NVIDIA
|
||||
bool _nvidia_available;
|
||||
std::shared_ptr<::streamfx::nvidia::cuda::obs> _nvcuda;
|
||||
std::shared_ptr<::streamfx::nvidia::cv::cv> _nvcvi;
|
||||
std::shared_ptr<::streamfx::nvidia::ar::ar> _nvar;
|
||||
#endif
|
||||
|
||||
public:
|
||||
autoframing_factory();
|
||||
~autoframing_factory() override;
|
||||
|
||||
const char* get_name() override;
|
||||
|
||||
void get_defaults2(obs_data_t* data) override;
|
||||
obs_properties_t* get_properties2(autoframing_instance* data) override;
|
||||
|
||||
#ifdef ENABLE_FRONTEND
|
||||
static bool on_manual_open(obs_properties_t* props, obs_property_t* property, void* data);
|
||||
#endif
|
||||
|
||||
bool is_provider_available(tracking_provider);
|
||||
tracking_provider find_ideal_provider();
|
||||
|
||||
public: // Singleton
|
||||
static void initialize();
|
||||
static void finalize();
|
||||
static std::shared_ptr<autoframing_factory> get();
|
||||
};
|
||||
} // namespace streamfx::filter::autoframing
|
||||
|
|
@ -35,6 +35,9 @@
|
|||
#include "encoders/encoder-ffmpeg.hpp"
|
||||
#endif
|
||||
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING
|
||||
#include "filters/filter-autoframing.hpp"
|
||||
#endif
|
||||
#ifdef ENABLE_FILTER_BLUR
|
||||
#include "filters/filter-blur.hpp"
|
||||
#endif
|
||||
|
|
@ -146,6 +149,9 @@ try {
|
|||
|
||||
// Filters
|
||||
{
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING
|
||||
streamfx::filter::autoframing::autoframing_factory::initialize();
|
||||
#endif
|
||||
#ifdef ENABLE_FILTER_BLUR
|
||||
streamfx::filter::blur::blur_factory::initialize();
|
||||
#endif
|
||||
|
|
@ -238,6 +244,9 @@ try {
|
|||
|
||||
// Filters
|
||||
{
|
||||
#ifdef ENABLE_FILTER_AUTOFRAMING
|
||||
streamfx::filter::autoframing::autoframing_factory::finalize();
|
||||
#endif
|
||||
#ifdef ENABLE_FILTER_BLUR
|
||||
streamfx::filter::blur::blur_factory::finalize();
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in a new issue