encoders/ffmpeg: Fix fallbacks, hide handler-less encoders and clean up

* Changes the encoder name to 'streamfx-{name}' from 'streamfx--{name}' as the latter is a typo, but adds a proxy to still support the latter in bad configurations.
* Some of the warning messages have been improved in order to better support end-users, and support for the new encoder error messages has been added.
* Adds support for the is_hw argument instead of blindly relying on obs_encoder_get_caps() which actually returns the wrong values due to rerouting.
* Fixed handler-less encoders showing up in the UI outside of debug builds.

source/encoders/encoder-ffmpeg.cpp

@@ -77,8 +77,8 @@ using namespace streamfx::encoder::codec;
 
 enum class keyframe_type { SECONDS, FRAMES };
 
-ffmpeg_instance::ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self)
+ffmpeg_instance::ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self, bool is_hw)
-	: encoder_instance(settings, self),
+	: encoder_instance(settings, self, is_hw),
 
 	  _factory(reinterpret_cast<ffmpeg_factory*>(obs_encoder_get_type_data(self))),
 
@@ -93,11 +93,11 @@ ffmpeg_instance::ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self)
 	  _free_frames(), _used_frames(), _free_frames_last_used()
 {
 	// Initialize GPU Stuff
-	if (obs_encoder_get_caps(self) & OBS_ENCODER_CAP_PASS_TEXTURE) {
+	if (is_hw) {
 		// Abort if user specified manual override.
 		if ((static_cast<AVPixelFormat>(obs_data_get_int(settings, KEY_FFMPEG_COLORFORMAT)) != AV_PIX_FMT_NONE)
 			|| (obs_data_get_int(settings, KEY_FFMPEG_GPU) != -1) || (obs_encoder_scaling_enabled(_self))) {
-			throw std::runtime_error("Unable to create accelerated encoder due to user settings.");
+			throw std::runtime_error("Selected settings prevent the use of hardware encoding, falling back to software.");
 		}
 
 #ifdef WIN32
@@ -125,7 +125,7 @@ ffmpeg_instance::ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self)
 	av_new_packet(&_packet, 8 * 1024 * 1024); // 8 MB precached Packet size.
 
 	// Initialize
-	if (obs_encoder_get_caps(self) & OBS_ENCODER_CAP_PASS_TEXTURE) {
+	if (is_hw) {
 		initialize_hw(settings);
 	} else {
 		initialize_sw(settings);
@@ -883,10 +883,26 @@ void ffmpeg_instance::parse_ffmpeg_commandline(std::string text)
 
 ffmpeg_factory::ffmpeg_factory(const AVCodec* codec) : _avcodec(codec)
 {
-	{ // Generate information
+	// Generate default identifier.
-		_id = std::string(PREFIX) + "-" + std::string(_avcodec->name);
+	{
+		std::stringstream str;
+		str << PREFIX << _avcodec->name;
+		_id = str.str();
+	}
 
-		// Figure out what codec this encoder is for.
+	{ // Generate default name.
+		std::stringstream str;
+		if (_avcodec->long_name) {
+			str << _avcodec->long_name;
+			str << " (" << _avcodec->name << ")";
+		} else {
+			str << _avcodec->name;
+		}
+		str << D_TRANSLATE(ST_FFMPEG_SUFFIX);
+		_name = str.str();
+	}
+
+	// Try and find a codec name that libOBS understands.
 	if (auto* desc = avcodec_descriptor_get(_avcodec->id); desc) {
 		_codec = desc->name;
 	} else {
@@ -894,24 +910,21 @@ ffmpeg_factory::ffmpeg_factory(const AVCodec* codec) : _avcodec(codec)
 		_codec = _avcodec->name;
 	}
 
-		// Generate a readable name
+	// Find any available handlers for this codec.
-		std::stringstream sstr;
+	if (_handler = ffmpeg_manager::get()->get_handler(_avcodec->name); _handler) {
-		if (_avcodec->long_name) {
+		// Override any found info with the one specified by the handler.
-			sstr << _avcodec->long_name;
-			sstr << " (" << _avcodec->name << ")";
-		} else {
-			sstr << _avcodec->name;
-		}
-		sstr << D_TRANSLATE(ST_FFMPEG_SUFFIX);
-		_name = sstr.str();
-	}
-
-	// Find Codec UI handler.
-	_handler = ffmpeg_manager::get()->get_handler(_avcodec->name);
-	if (_handler)
 		_handler->adjust_info(this, _avcodec, _id, _name, _codec);
 
-	// Build Info structure.
+		// Add texture capability for hardware encoders.
+		if (_handler->is_hardware_encoder(this)) {
+			_info.caps |= OBS_ENCODER_CAP_PASS_TEXTURE;
+		}
+	} else {
+		// If there are no handlers, default to mark it deprecated.
+		_info.caps |= OBS_ENCODER_CAP_DEPRECATED;
+	}
+
+	{ // Build Info structure.
 		_info.id    = _id.c_str();
 		_info.codec = _codec.c_str();
 		if (_avcodec->type == AVMediaType::AVMEDIA_TYPE_VIDEO) {
@@ -919,22 +932,21 @@ ffmpeg_factory::ffmpeg_factory(const AVCodec* codec) : _avcodec(codec)
 		} else if (_avcodec->type == AVMediaType::AVMEDIA_TYPE_AUDIO) {
 			_info.type = obs_encoder_type::OBS_ENCODER_AUDIO;
 		}
-	if (::ffmpeg::tools::can_hardware_encode(_avcodec)) {
-		_info.caps |= OBS_ENCODER_CAP_PASS_TEXTURE;
 	}
 
+	// Register encoder and proxies.
 	finish_setup();
-	{
+	const std::string proxies[] = {
-		std::string id  = std::string("StreamFX-") + _avcodec->name;
+		std::string("streamfx--") + _avcodec->name,
-		std::string id2 = id + "_sw";
+		std::string("StreamFX-") + _avcodec->name,
-		register_proxy(id);
+		std::string("obs-ffmpeg-encoder_") + _avcodec->name,
-		register_proxy(id2);
+	};
+	for (auto proxy_id : proxies) {
+		register_proxy(proxy_id);
+		if (_info.caps & OBS_ENCODER_CAP_PASS_TEXTURE) {
+			std::string proxy_fallback_id = proxy_id + "_sw";
+			register_proxy(proxy_fallback_id);
 		}
-	{
-		std::string id  = std::string("obs-ffmpeg-encoder_") + _avcodec->name;
-		std::string id2 = id + "_sw";
-		register_proxy(id);
-		register_proxy(id2);
 	}
 }
 

source/encoders/encoder-ffmpeg.hpp

@@ -77,7 +77,7 @@ namespace streamfx::encoder::ffmpeg {
 		std::chrono::high_resolution_clock::time_point _free_frames_last_used;
 
 		public:
-		ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self);
+		ffmpeg_instance(obs_data_t* settings, obs_encoder_t* self, bool is_hw);
 		virtual ~ffmpeg_instance();
 
 		public: