ffmpeg-encoder: Implement additional support checks

source/encoders/ffmpeg-encoder.cpp

@@ -512,7 +512,7 @@ void ffmpeg_factory::get_properties(obs_properties_t* props, bool hw_encode)
 	if (_handler)
 		_handler->get_properties(props, avcodec_ptr, nullptr, hw_encode);
 
-	if ((avcodec_ptr->capabilities & AV_CODEC_CAP_INTRA_ONLY) == 0) {
+	if (_handler && _handler->has_keyframe_support(this)) {
 		// Key-Frame Options
 		obs_properties_t* grp = props;
 		if (!util::are_property_groups_broken()) {
@@ -520,7 +520,7 @@ void ffmpeg_factory::get_properties(obs_properties_t* props, bool hw_encode)
 			obs_properties_add_group(props, ST_KEYFRAMES, D_TRANSLATE(ST_KEYFRAMES), OBS_GROUP_NORMAL, grp);
 		}
 
-		{
+		{ // Key-Frame Interval Type
 			auto p = obs_properties_add_list(grp, KEY_KEYFRAMES_INTERVALTYPE, D_TRANSLATE(ST_KEYFRAMES_INTERVALTYPE),
 											 OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
 			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_KEYFRAMES_INTERVALTYPE)));
@@ -528,13 +528,13 @@ void ffmpeg_factory::get_properties(obs_properties_t* props, bool hw_encode)
 			obs_property_list_add_int(p, D_TRANSLATE(ST_KEYFRAMES_INTERVALTYPE_(Seconds)), 0);
 			obs_property_list_add_int(p, D_TRANSLATE(ST_KEYFRAMES_INTERVALTYPE_(Frames)), 1);
 		}
-		{
+		{ // Key-Frame Interval Seconds
 			auto p = obs_properties_add_float(grp, KEY_KEYFRAMES_INTERVAL_SECONDS, D_TRANSLATE(ST_KEYFRAMES_INTERVAL),
 											  0.00, std::numeric_limits<std::int16_t>::max(), 0.01);
 			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_KEYFRAMES_INTERVAL)));
 			obs_property_float_set_suffix(p, " seconds");
 		}
-		{
+		{ // Key-Frame Interval Frames
 			auto p = obs_properties_add_int(grp, KEY_KEYFRAMES_INTERVAL_FRAMES, D_TRANSLATE(ST_KEYFRAMES_INTERVAL), 0,
 											std::numeric_limits<std::int32_t>::max(), 1);
 			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_KEYFRAMES_INTERVAL)));
@@ -550,34 +550,34 @@ void ffmpeg_factory::get_properties(obs_properties_t* props, bool hw_encode)
 			grp = prs;
 		}
 
-		{
+		{ // Custom Settings
 			auto p = obs_properties_add_text(grp, KEY_FFMPEG_CUSTOMSETTINGS, D_TRANSLATE(ST_FFMPEG_CUSTOMSETTINGS),
 											 obs_text_type::OBS_TEXT_DEFAULT);
 			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_CUSTOMSETTINGS)));
 		}
-		if (!hw_encode) {
+
-			{
+		if (_handler && _handler->is_hardware_encoder(this)) {
-				auto p = obs_properties_add_int(grp, KEY_FFMPEG_GPU, D_TRANSLATE(ST_FFMPEG_GPU), -1,
+			auto p = obs_properties_add_int(grp, KEY_FFMPEG_GPU, D_TRANSLATE(ST_FFMPEG_GPU), -1,
-												std::numeric_limits<std::uint8_t>::max(), 1);
+											std::numeric_limits<std::uint8_t>::max(), 1);
-				obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_GPU)));
+			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_GPU)));
-			}
+		}
-			if (avcodec_ptr->pix_fmts) {
+
-				auto p = obs_properties_add_list(grp, KEY_FFMPEG_COLORFORMAT, D_TRANSLATE(ST_FFMPEG_COLORFORMAT),
+		if (_handler && _handler->has_threading_support(this)) {
-												 OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
+			auto p = obs_properties_add_int_slider(grp, KEY_FFMPEG_THREADS, D_TRANSLATE(ST_FFMPEG_THREADS), 0,
-				obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_COLORFORMAT)));
+												   static_cast<int64_t>(std::thread::hardware_concurrency() * 2), 1);
-				obs_property_list_add_int(p, D_TRANSLATE(S_STATE_AUTOMATIC), static_cast<int64_t>(AV_PIX_FMT_NONE));
+			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_THREADS)));
-				for (auto ptr = avcodec_ptr->pix_fmts; *ptr != AV_PIX_FMT_NONE; ptr++) {
+		}
-					obs_property_list_add_int(p, ::ffmpeg::tools::get_pixel_format_name(*ptr),
+
-											  static_cast<int64_t>(*ptr));
+		if (_handler && _handler->has_pixel_format_support(this)) {
-				}
+			auto p = obs_properties_add_list(grp, KEY_FFMPEG_COLORFORMAT, D_TRANSLATE(ST_FFMPEG_COLORFORMAT),
-			}
+											 OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
-			if (avcodec_ptr->capabilities & (AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) {
+			obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_COLORFORMAT)));
-				auto p =
+			obs_property_list_add_int(p, D_TRANSLATE(S_STATE_AUTOMATIC), static_cast<int64_t>(AV_PIX_FMT_NONE));
-					obs_properties_add_int_slider(grp, KEY_FFMPEG_THREADS, D_TRANSLATE(ST_FFMPEG_THREADS), 0,
+			for (auto ptr = avcodec_ptr->pix_fmts; *ptr != AV_PIX_FMT_NONE; ptr++) {
-												  static_cast<int64_t>(std::thread::hardware_concurrency() * 2), 1);
+				obs_property_list_add_int(p, ::ffmpeg::tools::get_pixel_format_name(*ptr), static_cast<int64_t>(*ptr));
-				obs_property_set_long_description(p, D_TRANSLATE(D_DESC(ST_FFMPEG_THREADS)));
 			}
 		}
+
 		{
 			auto p =
 				obs_properties_add_list(grp, KEY_FFMPEG_STANDARDCOMPLIANCE, D_TRANSLATE(ST_FFMPEG_STANDARDCOMPLIANCE),
@@ -699,7 +699,7 @@ void ffmpeg_instance::initialize_hw(obs_data_t*)
 
 	_context->hw_frames_ctx = av_hwframe_ctx_alloc(_context->hw_device_ctx);
 	if (!_context->hw_frames_ctx)
-		throw std::runtime_error("Failed to allocate AVHWFramesContext.");
+		throw std::runtime_error("Allocating hardware context failed, chosen pixel format is likely not supported.");
 
 	AVHWFramesContext* ctx = reinterpret_cast<AVHWFramesContext*>(_context->hw_frames_ctx->data);
 	ctx->width             = _context->width;
@@ -708,7 +708,7 @@ void ffmpeg_instance::initialize_hw(obs_data_t*)
 	ctx->sw_format         = _context->sw_pix_fmt;
 
 	if (av_hwframe_ctx_init(_context->hw_frames_ctx) < 0)
-		throw std::runtime_error("Failed to initialize AVHWFramesContext.");
+		throw std::runtime_error("Initializing hardware context failed, chosen pixel format is likely not supported.");
 }
 
 void ffmpeg_instance::push_free_frame(std::shared_ptr<AVFrame> frame)
@@ -901,7 +901,7 @@ bool ffmpeg_instance::update(obs_data_t* settings)
 	}
 
 	// Keyframes
-	if (_handler && _handler->has_keyframe_support(this)) {
+	if (_handler && _handler->has_keyframe_support(_factory)) {
 		// Key-Frame Options
 		obs_video_info ovi;
 		if (!obs_get_video_info(&ovi)) {

source/encoders/handlers/handler.cpp

@@ -28,11 +28,26 @@ void encoder::ffmpeg::handler::handler::adjust_encoder_info(encoder::ffmpeg::ffm
 
 void encoder::ffmpeg::handler::handler::get_defaults(obs_data_t*, const AVCodec*, AVCodecContext*, bool) {}
 
-bool encoder::ffmpeg::handler::handler::has_keyframe_support(ffmpeg_instance* instance)
+bool encoder::ffmpeg::handler::handler::has_keyframe_support(ffmpeg_factory* instance)
 {
 	return (instance->get_avcodec()->capabilities & AV_CODEC_CAP_INTRA_ONLY) == 0;
 }
 
+bool encoder::ffmpeg::handler::handler::is_hardware_encoder(ffmpeg_factory* instance)
+{
+	return false;
+}
+
+bool encoder::ffmpeg::handler::handler::has_threading_support(ffmpeg_factory* instance)
+{
+	return (instance->get_avcodec()->capabilities & (AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS));
+}
+
+bool encoder::ffmpeg::handler::handler::has_pixel_format_support(ffmpeg_factory* instance)
+{
+	return (instance->get_avcodec()->pix_fmts != nullptr);
+}
+
 void encoder::ffmpeg::handler::handler::get_properties(obs_properties_t*, const AVCodec*, AVCodecContext*, bool) {}
 
 void encoder::ffmpeg::handler::handler::update(obs_data_t*, const AVCodec*, AVCodecContext*) {}

source/encoders/handlers/handler.hpp

@@ -46,9 +46,16 @@ namespace encoder::ffmpeg {
 			virtual void get_defaults(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context,
 									  bool hw_encode);
 
-			public /*settings*/:
+			public /*support tests*/:
-			virtual bool has_keyframe_support(ffmpeg_instance* instance);
+			virtual bool has_keyframe_support(ffmpeg_factory* instance);
 
+			virtual bool is_hardware_encoder(ffmpeg_factory* instance);
+
+			virtual bool has_threading_support(ffmpeg_factory* instance);
+
+			virtual bool has_pixel_format_support(ffmpeg_factory* instance);
+
+			public /*settings*/:
 			virtual void get_properties(obs_properties_t* props, const AVCodec* codec, AVCodecContext* context,
 										bool hw_encode);
 

source/encoders/handlers/nvenc_h264_handler.cpp

@@ -75,6 +75,21 @@ void nvenc_h264_handler::get_defaults(obs_data_t* settings, const AVCodec* codec
 	obs_data_set_default_int(settings, KEY_LEVEL, static_cast<int64_t>(level::UNKNOWN));
 }
 
+bool encoder::ffmpeg::handler::nvenc_h264_handler::is_hardware_encoder(ffmpeg_factory* instance)
+{
+	return true;
+}
+
+bool encoder::ffmpeg::handler::nvenc_h264_handler::has_threading_support(ffmpeg_factory* instance)
+{
+	return false;
+}
+
+bool encoder::ffmpeg::handler::nvenc_h264_handler::has_pixel_format_support(ffmpeg_factory* instance)
+{
+	return true;
+}
+
 bool nvenc_h264_handler::has_keyframe_support(ffmpeg_instance*)
 {
 	return true;

source/encoders/handlers/nvenc_h264_handler.hpp

@@ -39,9 +39,16 @@ namespace encoder::ffmpeg::handler {
 
 		virtual void get_defaults(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context, bool hw_encode);
 
-		public /*settings*/:
+		public /*support tests*/:
 		virtual bool has_keyframe_support(ffmpeg_instance* instance);
 
+		virtual bool is_hardware_encoder(ffmpeg_factory* instance);
+
+		virtual bool has_threading_support(ffmpeg_factory* instance);
+
+		virtual bool has_pixel_format_support(ffmpeg_factory* instance);
+
+		public /*settings*/:
 		virtual void get_properties(obs_properties_t* props, const AVCodec* codec, AVCodecContext* context,
 									bool hw_encode);
 

source/encoders/handlers/nvenc_shared.cpp

@@ -574,19 +574,25 @@ void nvenc::update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* c
 			break;
 		}
 
+		// Two Pass
 		if (int tp = static_cast<int>(obs_data_get_int(settings, KEY_RATECONTROL_TWOPASS)); tp > -1) {
 			av_opt_set_int(context->priv_data, "2pass", tp ? 1 : 0, AV_OPT_SEARCH_CHILDREN);
 		}
 
+		// Look Ahead # of Frames
 		int la = static_cast<int>(obs_data_get_int(settings, KEY_RATECONTROL_LOOKAHEAD));
-		if (la > -1)
+		if (!util::is_tristate_default(la)) {
 			av_opt_set_int(context->priv_data, "rc-lookahead", la, AV_OPT_SEARCH_CHILDREN);
+		}
+
 		if (la > 0) {
+			// Adaptive I-Frames
 			if (int64_t adapt_i = obs_data_get_int(settings, KEY_RATECONTROL_ADAPTIVEI);
 				!util::is_tristate_default(adapt_i)) {
 				av_opt_set_int(context->priv_data, "no-scenecut", adapt_i, AV_OPT_SEARCH_CHILDREN);
 			}
 
+			// Adaptive B-Frames
 			if (strcmp(codec->name, "h264_nvenc")) {
 				if (int64_t adapt_b = obs_data_get_int(settings, KEY_RATECONTROL_ADAPTIVEB);
 					!util::is_tristate_default(adapt_b)) {
@@ -618,6 +624,8 @@ void nvenc::update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* c
 			//context->rc_min_rate = 0;
 			context->rc_max_rate = 0;
 		}
+
+		// Buffer Size
 		if (have_bitrate || have_bitrate_range) {
 			if (int64_t v = obs_data_get_int(settings, KEY_RATECONTROL_BUFFERSIZE); v > -1)
 				context->rc_buffer_size = static_cast<int>(v * 1000);
@@ -625,6 +633,7 @@ void nvenc::update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* c
 			context->rc_buffer_size = 0;
 		}
 
+		// Quality Limits
 		if (have_quality) {
 			if (int qmin = static_cast<int>(obs_data_get_int(settings, KEY_RATECONTROL_QUALITY_MINIMUM)); qmin > -1)
 				context->qmin = qmin;
@@ -635,12 +644,12 @@ void nvenc::update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* c
 			context->qmax = -1;
 		}
 
-		{
+		// Quality Target
-			if (double_t v = obs_data_get_double(settings, KEY_RATECONTROL_QUALITY_TARGET) / 100.0 * 51.0; v > 0) {
+		if (double_t v = obs_data_get_double(settings, KEY_RATECONTROL_QUALITY_TARGET) / 100.0 * 51.0; v > 0) {
-				av_opt_set_double(context->priv_data, "cq", v, AV_OPT_SEARCH_CHILDREN);
+			av_opt_set_double(context->priv_data, "cq", v, AV_OPT_SEARCH_CHILDREN);
-			}
 		}
 
+		// QP Settings
 		if (have_qp) {
 			if (int64_t qp = obs_data_get_int(settings, KEY_RATECONTROL_QP_I); qp > -1)
 				av_opt_set_int(context->priv_data, "init_qpI", static_cast<int>(qp), AV_OPT_SEARCH_CHILDREN);

source/encoders/handlers/prores_aw_handler.cpp

@@ -60,6 +60,11 @@ void prores_aw_handler::get_defaults(obs_data_t* settings, const AVCodec*, AVCod
 	obs_data_set_default_int(settings, P_PRORES_PROFILE, 0);
 }
 
+bool encoder::ffmpeg::handler::prores_aw_handler::has_pixel_format_support(ffmpeg_factory* instance)
+{
+	return false;
+}
+
 inline const char* profile_to_name(const AVProfile* ptr)
 {
 	switch (static_cast<encoder::codec::prores::profile>(ptr->profile)) {

source/encoders/handlers/prores_aw_handler.hpp

@@ -34,13 +34,14 @@ namespace encoder::ffmpeg::handler {
 		public:
 		virtual ~prores_aw_handler(){};
 
-		public:
+		public /*factory*/:
-		virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings, const AVCodec* codec,
-										  AVCodecContext* context) override;
-
 		virtual void get_defaults(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context,
 								  bool hw_encode) override;
 
+		public /*support tests*/:
+		virtual bool has_pixel_format_support(ffmpeg_factory* instance);
+
+		public /*settings*/:
 		virtual void get_properties(obs_properties_t* props, const AVCodec* codec, AVCodecContext* context,
 									bool hw_encode) override;
 
@@ -48,6 +49,10 @@ namespace encoder::ffmpeg::handler {
 
 		virtual void log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) override;
 
+		public /*instance*/:
+		virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings, const AVCodec* codec,
+										  AVCodecContext* context) override;
+
 		virtual void process_avpacket(AVPacket& packet, const AVCodec* codec, AVCodecContext* context) override;
 	};
 } // namespace encoder::ffmpeg::handler