gs-mipmapper: Early attempts at proper mipmap rendering

This class is an attempt at adding dynamic mipmapping support to OBS, which is lacking this feature. It is pretty much a hack until I figure out how to do it for both d3d11 and opengl and can make a PR for obs-studio to include this ability by default.

CMakeLists.txt

@@ -201,6 +201,7 @@ SET(PROJECT_DATA_EFFECTS
 	"${PROJECT_SOURCE_DIR}/data/effects/displace.effect"
 	"${PROJECT_SOURCE_DIR}/data/effects/color-conversion.effect"
 	"${PROJECT_SOURCE_DIR}/data/effects/mip-mapper.effect"
+	"${PROJECT_SOURCE_DIR}/data/effects/mipgen.effect"
 )
 SET(PROJECT_DATA_SHADERS
 #	"${PROJECT_SOURCE_DIR}/data/shaders/name.effect"
@@ -242,7 +243,8 @@ SET(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/gs-indexbuffer.h"
 	"${PROJECT_SOURCE_DIR}/source/gs-indexbuffer.cpp"
 	"${PROJECT_SOURCE_DIR}/source/gs-limits.h"
-#	"${PROJECT_SOURCE_DIR}/source/gs-mipmapper.h"
+	"${PROJECT_SOURCE_DIR}/source/gs-mipmapper.h"
+	"${PROJECT_SOURCE_DIR}/source/gs-mipmapper.cpp"
 	"${PROJECT_SOURCE_DIR}/source/gs-rendertarget.h"
 	"${PROJECT_SOURCE_DIR}/source/gs-rendertarget.cpp"
 	"${PROJECT_SOURCE_DIR}/source/gs-sampler.h"
@@ -263,7 +265,6 @@ SET(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/util-math.cpp"
 	"${PROJECT_SOURCE_DIR}/source/util-memory.h"
 	"${PROJECT_SOURCE_DIR}/source/util-memory.cpp"
-#	"${PROJECT_SOURCE_DIR}/source/gs-mipmapper.cpp"
 )
 
 Source_Group("Data Files\\Locale" FILES ${PROJECT_DATA_LOCALE})
@@ -350,13 +351,13 @@ If (WIN32)
 			NODRAWTEXT
 			NOGDI
 			NOKERNEL
-			NOUSER
+			#NOUSER
 			NONLS
 			NOMB
 			NOMEMMGR
 			NOMETAFILE
 			NOMINMAX
-			NOMSG
+			#NOMSG
 			NOOPENFILE
 			NOSCROLL
 			NOSERVICE

source/gs-mipmapper.cpp

@@ -18,142 +18,200 @@
  */
 
 #include "gs-mipmapper.h"
+
 extern "C" {
-#pragma warning (push)
+#pragma warning(push)
-#pragma warning (disable: 4201)
+#pragma warning(disable : 4201)
-#include "libobs/obs.h"
+#include <graphics/graphics.h>
-#include "libobs/graphics/graphics.h"
+#include <obs-module.h>
-#pragma warning (pop)
+#include <obs.h>
+#pragma warning(pop)
+#if defined(WIN32) || defined(WIN64)
+#include <windows.h>
+#endif
 }
 
-#define MAX_LAYERS 16
+// Here be dragons!
+// This is to add support for mipmap generation which is by default not possible with libobs.
+// OBS hides a ton of possible things from us, which we'd have to simulate - or just hack around.
+struct graphics_subsystem {
+	void*        module;
+	gs_device_t* device;
+	// No other fields required.
+};
 
-GS::MipMapper::MipMapper() : m_rt(GS_RGBA, GS_ZS_NONE), m_vb(65535) {
+#if defined(WIN32) || defined(WIN64)
-	// Sub layers start here now.
+#include <d3d11.h>
-	float_t size = 0.5, offset = 0.0;
+#include <dxgi.h>
-	for (size_t layer = 0; layer < MAX_LAYERS; layer++) {
+#include <util/windows/ComPtr.hpp>
-		// Large Block
-		if (layer == 0) {
-			{
-				GS::Vertex& v = m_vb.at(layer * 12);
-				vec3_set(&v.position, offset, offset, 0);
-				vec4_set(&v.uv[0], 0, 0, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 1);
-				vec3_set(&v.position, offset + size, offset, 0);
-				vec4_set(&v.uv[0], 1, 0, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 2);
-				vec3_set(&v.position, offset + size, offset + size, 0);
-				vec4_set(&v.uv[0], 1, 1, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 3);
-				vec3_set(&v.position, offset, offset + size, 0);
-				vec4_set(&v.uv[0], 0, 1, 0, 0);
-			}
-		} else {
-			{
-				GS::Vertex& v = m_vb.at(layer * 12);
-				vec3_set(&v.position, offset, offset, 0);
-				vec4_set(&v.uv[0], offset - size * 2, offset - size * 2, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 1);
-				vec3_set(&v.position, offset + size, offset, 0);
-				vec4_set(&v.uv[0], offset, offset - size * 2, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 2);
-				vec3_set(&v.position, offset + size, offset + size, 0);
-				vec4_set(&v.uv[0], offset, offset, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 3);
-				vec3_set(&v.position, offset, offset + size, 0);
-				vec4_set(&v.uv[0], offset - size * 2, offset, 0, 0);
-			}
-		}
 
-		if (layer != 0) {
+// Slaughtered copy of d3d11-subsystem.hpp gs_device. We only need up to device and context, the rest is "unknown" to us.
-			// Horizontal Slice
+struct gs_d3d11_device {
-			{
+	ComPtr<IDXGIFactory1>       factory;
-				GS::Vertex& v = m_vb.at(layer * 12 + 4);
+	ComPtr<IDXGIAdapter1>       adapter;
-				vec3_set(&v.position, 0, offset, 0);
+	ComPtr<ID3D11Device>        device;
-				vec4_set(&v.uv[0], 0, offset - size, 0, 0);
+	ComPtr<ID3D11DeviceContext> context;
-			}
+	// No other fields required.
-			{
+};
-				GS::Vertex& v = m_vb.at(layer * 12 + 5);
+#endif
-				vec3_set(&v.position, size, offset, 0);
-				vec4_set(&v.uv[0], .5, offset - size, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 6);
-				vec3_set(&v.position, size, offset + size, 0);
-				vec4_set(&v.uv[0], .5, offset, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 7);
-				vec3_set(&v.position, 0, offset + size, 0);
-				vec4_set(&v.uv[0], 0, offset, 0, 0);
-			}
 
-			// Vertical Slice
+gs::mipmapper::~mipmapper()
-			{
+{
-				GS::Vertex& v = m_vb.at(layer * 12 + 8);
+	vertexBuffer.reset();
-				vec3_set(&v.position, offset, 0, 0);
+	renderTarget.reset();
-				vec4_set(&v.uv[0], offset - size, 0, 0, 0);
+	effect.reset();
-			}
+}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 9);
-				vec3_set(&v.position, offset + size, 0, 0);
-				vec4_set(&v.uv[0], offset, 0, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 10);
-				vec3_set(&v.position, offset + size, size, 0);
-				vec4_set(&v.uv[0], offset, 0.5, 0, 0);
-			}
-			{
-				GS::Vertex& v = m_vb.at(layer * 12 + 11);
-				vec3_set(&v.position, offset, size, 0);
-				vec4_set(&v.uv[0], offset - size, 0.5, 0, 0);
-			}
-		}
 
-		offset += size;
+gs::mipmapper::mipmapper()
-		size /= 2.0;
+{
+	vertexBuffer   = std::make_unique<gs::vertex_buffer>(6, 1);
+	auto v0        = vertexBuffer->at(0);
+	v0.position->x = 0;
+	v0.position->y = 0;
+	v0.uv[0]->x    = 0;
+	v0.uv[0]->y    = 0;
+
+	auto v1        = vertexBuffer->at(1);
+	auto v4        = vertexBuffer->at(4);
+	v4.position->x = v1.position->x = 1.0;
+	v4.position->y = v1.position->y = 0;
+	v4.uv[0]->x = v1.uv[0]->x = 1.0;
+	v4.uv[0]->y = v1.uv[0]->y = 0;
+
+	auto v2        = vertexBuffer->at(2);
+	auto v3        = vertexBuffer->at(3);
+	v3.position->x = v2.position->x = 0;
+	v3.position->y = v2.position->y = 1.0;
+	v3.uv[0]->x = v2.uv[0]->x = 0;
+	v3.uv[0]->y = v2.uv[0]->y = 1.0;
+
+	auto v5        = vertexBuffer->at(5);
+	v5.position->x = 1.0;
+	v5.position->y = 1.0;
+	v5.uv[0]->x    = 1.0;
+	v5.uv[0]->y    = 1.0;
+
+	vertexBuffer->update();
+
+	char* effect_file = obs_module_file("effects/mipgen.effect");
+	effect            = std::make_unique<gs::effect>(effect_file);
+	bfree(effect_file);
+}
+
+void gs::mipmapper::rebuild(std::shared_ptr<gs::texture> texture,
+							gs::mipmapper::generator     generator = gs::mipmapper::generator::Bilinear,
+							float_t                      strength  = 1.0)
+{
+	// Here be dragons! You have been warned.
+
+	// Do nothing if there is no texture given.
+	if (!texture) {
+#ifdef _DEBUG
+		assert(!texture);
+#endif
+		return;
 	}
-}
 
-GS::MipMapper::~MipMapper() {}
+	obs_enter_graphics();
 
-gs_texture_t* GS::MipMapper::Render(gs_texture_t* intex) {
+	// Test if we actually need to recreate the render target for a different format.
-	// Get Texture Size
+	bool recreate = false;
-	uint32_t width = gs_texture_get_width(intex),
+	if (renderTarget) {
-		height = gs_texture_get_height(intex);
+		recreate = (texture->get_color_format() != renderTarget->get_color_format());
+	} else {
+		recreate = true;
+	}
 
-	// Calculate total mipmap layers.
+	// Re-create the render target if necessary.
-	uint32_t layers = 1, size = height;
+	if (recreate) {
-	if (width > height) size = width;
+		renderTarget = std::make_unique<gs::rendertarget>(texture->get_color_format(), GS_ZS_NONE);
-	layers = static_cast<uint32_t>(ceil(log(double_t(size)) / log(2.0)));
+	}
-	size = (uint32_t)pow(2, layers);
 
 	// Render
-	m_buf = GS::Texture(size, size, GS_RGBA, 1, nullptr, GS_DYNAMIC);
+	graphics_t*      ctx         = gs_get_context();
-	for (uint32_t layer = 0; layer < layers; layer++) {
+	gs_d3d11_device* dev         = reinterpret_cast<gs_d3d11_device*>(ctx->device);
-		try {
+	int              device_type = gs_get_device_type();
-			GS::RenderTargetOp op = m_rt.Render(size * 2, size * 2);
+	void*            obj         = gs_texture_get_obj(texture->get_object());
-			gs_load_vertexbuffer(m_vb.get());
+	std::string      technique   = "Draw";
-			gs_load_indexbuffer(nullptr);
+
-			gs_draw(gs_draw_mode::GS_TRIS, layer * 12, 12);
+	switch (generator) {
-		} catch (...) {
+	case generator::Point:
-			return intex;
+		technique = "Point";
-		}
+		break;
-		gs_copy_texture(m_buf.GetObject(), m_rt.GetTextureObject());
+	case generator::Linear:
+		technique = "Linear";
+		break;
+	case generator::Bilinear:
+		technique = "Bilinear";
+		break;
+	case generator::Sharpen:
+		technique = "Sharpen";
+		break;
+	case generator::Smoothen:
+		technique = "Smoothen";
+		break;
+	case generator::Bicubic:
+		technique = "Bicubic";
+		break;
+	case generator::Lanczos:
+		technique = "Lanczos";
+		break;
 	}
 
-	return m_buf.GetObject();
+	gs_load_vertexbuffer(vertexBuffer->update());
+	gs_load_indexbuffer(nullptr);
+
+#if defined(WIN32) || defined(WIN64)
+	if (device_type == GS_DEVICE_DIRECT3D_11) {
+		// We definitely have a Direct3D11 resource.
+		if (texture->get_type() == gs::texture::type::Normal) {
+			D3D11_TEXTURE2D_DESC t2dsc;
+			ID3D11Texture2D*     t2 = reinterpret_cast<ID3D11Texture2D*>(obj);
+
+			t2->GetDesc(&t2dsc);
+
+			// If we do not have any miplevels, just stop now.
+			if (t2dsc.MipLevels == 0) {
+				obs_leave_graphics();
+				return;
+			}
+
+			size_t  texture_width  = texture->get_width();
+			size_t  texture_height = texture->get_height();
+			float_t texel_width    = 1.0 / texture_width;
+			float_t texel_height   = 1.0 / texture_height;
+
+			for (size_t mip = 1; mip < t2dsc.MipLevels; mip++) {
+				texture_width /= 2;
+				texture_height /= 2;
+
+				// Draw mipmap layer
+				{
+					auto op = renderTarget->render(texture_width, texture_height);
+
+					effect->get_parameter("image").set_texture(texture);
+					effect->get_parameter("imageTexel").set_float2(texel_width, texel_height);
+					effect->get_parameter("strength").set_float(strength);
+
+					while (gs_effect_loop(effect->get_object(), technique.c_str())) {
+						gs_draw(gs_draw_mode::GS_TRIS, 0, vertexBuffer->size());
+					}
+				}
+
+				// Copy
+				ID3D11Texture2D* rt =
+					reinterpret_cast<ID3D11Texture2D*>(gs_texture_get_obj(renderTarget->get_object()));
+				dev->context->CopySubresourceRegion(t2, mip, 0, 0, 0, rt, 0, nullptr);
+			}
+		}
+	}
+#endif
+	if (device_type == GS_DEVICE_OPENGL) {
+		// This is an OpenGL resource.
+	}
+
+	gs_load_indexbuffer(nullptr);
+	gs_load_vertexbuffer(nullptr);
+
+	obs_leave_graphics();
 }

source/gs-mipmapper.h

@@ -18,28 +18,39 @@
  */
 
 #pragma once
-#include "gs-vertexbuffer.h"
+#include "gs-effect.h"
 #include "gs-rendertarget.h"
 #include "gs-texture.h"
+#include "gs-vertexbuffer.h"
 
 extern "C" {
-	#pragma warning (push)
+#pragma warning(push)
-	#pragma warning (disable: 4201)
+#pragma warning(disable : 4201)
-	#include "libobs/graphics/graphics.h"
+#include <graphics/graphics.h>
-	#pragma warning (pop)
+#pragma warning(pop)
 }
 
-namespace GS {
+namespace gs {
-	class MipMapper {
+	class mipmapper {
+		std::unique_ptr<gs::vertex_buffer> vertexBuffer;
+		std::unique_ptr<gs::rendertarget>  renderTarget;
+		std::unique_ptr<gs::effect>        effect;
+
 		public:
-		MipMapper();
+		enum class generator : uint8_t {
-		~MipMapper();
+			Point,
+			Linear,
+			Bilinear,
+			Sharpen,
+			Smoothen,
+			Bicubic,
+			Lanczos,
+		};
 
-		gs_texture_t* Render(gs_texture_t*);
+		public:
+		~mipmapper();
+		mipmapper();
 
-		private:
+		void rebuild(std::shared_ptr<gs::texture> texture, gs::mipmapper::generator generator, float_t strength);
-		GS::VertexBuffer m_vb;
-		GS::RenderTarget m_rt;
-		GS::Texture m_buf;
 	};
-}
+} // namespace gs