From c7cfbff7dd80b1f089a4951cfbcdee71db299bdf Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Fri, 3 Nov 2017 00:13:07 +0100 Subject: [PATCH] gs-vertexbuffer: Workaround for a foreign deallocation bug in libobs The graphics subsystem in OBS is freeing memory it didn't allocate, resulting in stack/heap corruption and other kinds of messy situations. The official "fix" for this is to use bmalloc, but then you lose any kind of ability to re-use the same buffer for multiple vertex buffers or update on the fly without adjusting code that is possibly outside of your control (such as in libraries). This works around the issues by "patching" the gs_vertexbuffer object to no longer hold a reference to the gs_vb_data object. Currently only D3D11 is supported for this kind of hack and it might break in a future obs-studio release. PR fixing this Issue: https://github.com/jp9000/obs-studio/pull/993 --- source/gs-vertexbuffer.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/source/gs-vertexbuffer.cpp b/source/gs-vertexbuffer.cpp index be16adbb..3f9c988e 100644 --- a/source/gs-vertexbuffer.cpp +++ b/source/gs-vertexbuffer.cpp @@ -20,10 +20,10 @@ #include "gs-vertexbuffer.h" #include extern "C" { - #pragma warning( push ) - #pragma warning( disable: 4201 ) - #include - #pragma warning( pop ) +#pragma warning( push ) +#pragma warning( disable: 4201 ) +#include +#pragma warning( pop ) } const uint32_t defaultMaximumVertices = 65535u; @@ -78,6 +78,20 @@ GS::VertexBuffer::VertexBuffer(VertexBuffer& other) : VertexBuffer(other.m_maxim GS::VertexBuffer::~VertexBuffer() { if (m_vertexbuffer) { std::memset(&m_vertexbufferdata, 0, sizeof(m_vertexbufferdata)); + + // Workaround API stupidity in obs-studio. + // See PR: https://github.com/jp9000/obs-studio/pull/993 + + if (gs_get_device_type() == GS_DEVICE_DIRECT3D_11) { + #ifdef _DEBUG + *(reinterpret_cast(m_vertexbuffer) + 14) = 0; + #else + *(reinterpret_cast(m_vertexbuffer) + 13) = 0; + #endif + } else { + //*(reinterpret_cast(m_vertexbuffer)+14) = 0; + } + obs_enter_graphics(); gs_vertexbuffer_destroy(m_vertexbuffer); obs_leave_graphics();