From c2c5afd763cb6fcb3bc7a3de38931c22babd0893 Mon Sep 17 00:00:00 2001
From: Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
Date: Fri, 24 Apr 2020 06:06:46 +0200
Subject: [PATCH] gfx-shader: Actually write alpha channel to render target

---
 data/examples/shaders/filter/test.effect | 53 ++++++++++++++++++++++++
 data/examples/shaders/source/test.effect | 44 ++++++++++++++++++++
 source/gfx/shader/gfx-shader.cpp         | 10 ++++-
 3 files changed, 105 insertions(+), 2 deletions(-)
 create mode 100644 data/examples/shaders/filter/test.effect
 create mode 100644 data/examples/shaders/source/test.effect

diff --git a/data/examples/shaders/filter/test.effect b/data/examples/shaders/filter/test.effect
new file mode 100644
index 00000000..e958d0d4
--- /dev/null
+++ b/data/examples/shaders/filter/test.effect
@@ -0,0 +1,53 @@
+// This file is used to test features.
+
+uniform float4x4 ViewProj<
+	bool automatic = true;
+>;
+uniform float4 Time<
+	bool automatic = true;
+>;
+uniform texture2d InputA<
+	bool automatic = true;
+>;
+
+struct VertexData {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+sampler_state def_sampler {
+	AddressU  = Wrap;
+	AddressV  = Wrap;
+	Filter    = Linear;
+};
+
+VertexData VSDefault(VertexData vtx) {
+	vtx.pos = mul(float4(vtx.pos.xyz, 1.0), ViewProj);
+	return vtx;
+}
+
+float4 PSSolid(VertexData vtx) : TARGET {
+	return InputA.Sample(def_sampler, vtx.uv) * float4(cos(Time.y * 3.141 * 2.) * .5 + .5, cos(sin(Time.y * 3.141 * 2.) * 3.141), sin(Time.y * 3.141 * 2.) * .5 + .5, 1.);
+}
+
+technique Solid
+{
+	pass
+	{
+		vertex_shader = VSDefault(vtx);
+		pixel_shader  = PSSolid(vtx); 
+	}
+}
+
+float4 PSTranslucent(VertexData vtx) : TARGET {
+	return InputA.Sample(def_sampler, vtx.uv) * float4(1., 1., 1., sin(Time.y * 3.141 * 2.) * .5 + .5);
+}
+
+technique Translucent
+{
+	pass
+	{
+		vertex_shader = VSDefault(vtx);
+		pixel_shader  = PSTranslucent(vtx); 
+	}
+}
diff --git a/data/examples/shaders/source/test.effect b/data/examples/shaders/source/test.effect
new file mode 100644
index 00000000..f7f5fcf4
--- /dev/null
+++ b/data/examples/shaders/source/test.effect
@@ -0,0 +1,44 @@
+// This file is used to test features.
+
+uniform float4x4 ViewProj<
+	bool automatic = true;
+>;
+uniform float4 Time<
+	bool automatic = true;
+>;
+
+struct VertexData {
+	float4 pos : POSITION;
+	float2 uv  : TEXCOORD0;
+};
+
+VertexData VSDefault(VertexData vtx) {
+	vtx.pos = mul(float4(vtx.pos.xyz, 1.0), ViewProj);
+	return vtx;
+}
+
+float4 PSSolid(VertexData vtx) : TARGET {
+	return float4(cos(Time.y * 3.141 * 2.) * .5 + .5, cos(sin(Time.y * 3.141 * 2.) * 3.141), sin(Time.y * 3.141 * 2.) * .5 + .5, 1.);
+}
+
+technique Solid
+{
+	pass
+	{
+		vertex_shader = VSDefault(vtx);
+		pixel_shader  = PSSolid(vtx); 
+	}
+}
+
+float4 PSTranslucent(VertexData vtx) : TARGET {
+	return float4(1., 1., 1., sin(Time.y * 3.141 * 2.) * .5 + .5);
+}
+
+technique Translucent
+{
+	pass
+	{
+		vertex_shader = VSDefault(vtx);
+		pixel_shader  = PSTranslucent(vtx); 
+	}
+}
diff --git a/source/gfx/shader/gfx-shader.cpp b/source/gfx/shader/gfx-shader.cpp
index a28b31af..7d06b875 100644
--- a/source/gfx/shader/gfx-shader.cpp
+++ b/source/gfx/shader/gfx-shader.cpp
@@ -205,7 +205,7 @@ void gfx::shader::shader::properties(obs_properties_t* pr)
 				path = _shader_file.parent_path().string();
 			} else {
 				char* vp = obs_module_file("examples");
-				path = vp;
+				path     = vp;
 				bfree(vp);
 			}
 			auto p = obs_properties_add_path(grp, ST_SHADER_FILE, D_TRANSLATE(ST_SHADER_FILE), OBS_PATH_FILE, "*.*",
@@ -477,12 +477,18 @@ void gfx::shader::shader::render()
 		vec4 zero = {0, 0, 0, 0};
 		gs_ortho(0, width(), 0, height(), 0, 1);
 		gs_clear(GS_CLEAR_COLOR, &zero, 0, 0);
+
+		gs_blend_state_push();
+		gs_reset_blend_state();
+
 		gs_enable_blending(true);
-		gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO);
+		gs_blend_function_separate(GS_BLEND_SRCCOLOR, GS_BLEND_ZERO, GS_BLEND_SRCALPHA, GS_BLEND_ZERO);
 		gs_enable_color(true, true, true, true);
 		while (gs_effect_loop(_shader.get_object(), _shader_tech.c_str())) {
 			gs_draw_sprite(nullptr, 0, width(), height());
 		}
+
+		gs_blend_state_pop();
 	}
 
 	gs_effect_set_texture(gs_effect_get_param_by_name(obs_get_base_effect(OBS_EFFECT_DEFAULT), "image"),