From c2d261e77cccbfc4500a25ec276877dc5cb38079 Mon Sep 17 00:00:00 2001 From: Richard Stanway Date: Thu, 30 Jul 2026 22:03:03 +0200 Subject: [PATCH 1/2] libobs-d3d11: Fix crashes from stale vertex buffer data Crash reports indicated numerous crashes in LoadVertexBufferData. Looking through the code revealed that the device's curVertexBuffer / curVertexShader / curIndexBuffer variables were not cleared when the corresponding object was destroyed, leaving them pointing to dangling memory. This also affected the device rebuild path in reverse - cur was cleared, last was not. The buffer list was also assuming that every vertex shader has vertex positions, which is not true for the triangles drawn by async sources and GPU conversion textures. As those draws never load a vertex buffer, they ended up using whatever was left by the previous draw call - incorrect but harmless if the buffer is valid, use-after-free if it is not. --- libobs-d3d11/d3d11-rebuild.cpp | 2 ++ libobs-d3d11/d3d11-shader.cpp | 16 +++++++++++++--- libobs-d3d11/d3d11-subsystem.cpp | 15 +++++++++++++-- libobs-d3d11/d3d11-subsystem.hpp | 6 +++++- libobs-d3d11/d3d11-vertexbuffer.cpp | 4 +++- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/libobs-d3d11/d3d11-rebuild.cpp b/libobs-d3d11/d3d11-rebuild.cpp index 7f11c02e46b853..40b91f95ee04c5 100644 --- a/libobs-d3d11/d3d11-rebuild.cpp +++ b/libobs-d3d11/d3d11-rebuild.cpp @@ -575,6 +575,8 @@ try { curVertexShader = nullptr; curPixelShader = nullptr; curSwapChain = nullptr; + lastVertexBuffer = nullptr; + lastVertexShader = nullptr; zstencilStateChanged = true; rasterStateChanged = true; blendStateChanged = true; diff --git a/libobs-d3d11/d3d11-shader.cpp b/libobs-d3d11/d3d11-shader.cpp index 930062b099b49f..d13d0266acaece 100644 --- a/libobs-d3d11/d3d11-shader.cpp +++ b/libobs-d3d11/d3d11-shader.cpp @@ -32,7 +32,9 @@ void gs_vertex_shader::GetBuffersExpected(const std::vectordevice->lastVertexShader == shader) { - shader->device->lastVertexShader = nullptr; + if (shader) { + gs_device_t *device = shader->device; + if (device->curVertexShader == shader) { + device->curVertexShader = nullptr; + } + if (device->lastVertexShader == shader) { + device->lastVertexShader = nullptr; + } } + delete shader; } diff --git a/libobs-d3d11/d3d11-subsystem.cpp b/libobs-d3d11/d3d11-subsystem.cpp index c44761cdbe5e98..2ad0f4ef8c865b 100644 --- a/libobs-d3d11/d3d11-subsystem.cpp +++ b/libobs-d3d11/d3d11-subsystem.cpp @@ -2901,9 +2901,16 @@ void gs_samplerstate_destroy(gs_samplerstate_t *samplerstate) void gs_vertexbuffer_destroy(gs_vertbuffer_t *vertbuffer) { - if (vertbuffer && vertbuffer->device->lastVertexBuffer == vertbuffer) { - vertbuffer->device->lastVertexBuffer = nullptr; + if (vertbuffer) { + gs_device_t *device = vertbuffer->device; + if (device->curVertexBuffer == vertbuffer) { + device->curVertexBuffer = nullptr; + } + if (device->lastVertexBuffer == vertbuffer) { + device->lastVertexBuffer = nullptr; + } } + delete vertbuffer; } @@ -2956,6 +2963,10 @@ struct gs_vb_data *gs_vertexbuffer_get_data(const gs_vertbuffer_t *vertbuffer) void gs_indexbuffer_destroy(gs_indexbuffer_t *indexbuffer) { + if (indexbuffer && indexbuffer->device->curIndexBuffer == indexbuffer) { + indexbuffer->device->curIndexBuffer = nullptr; + } + delete indexbuffer; } diff --git a/libobs-d3d11/d3d11-subsystem.hpp b/libobs-d3d11/d3d11-subsystem.hpp index 3bae9d0e8858f9..3d33c2eeefd532 100644 --- a/libobs-d3d11/d3d11-subsystem.hpp +++ b/libobs-d3d11/d3d11-subsystem.hpp @@ -723,6 +723,7 @@ struct gs_vertex_shader : gs_shader { std::vector layoutData; + bool hasPositions; bool hasNormals; bool hasColors; bool hasTangents; @@ -739,7 +740,10 @@ struct gs_vertex_shader : gs_shader { inline uint32_t NumBuffersExpected() const { - uint32_t count = nTexUnits + 1; + uint32_t count = nTexUnits; + if (hasPositions) { + count++; + } if (hasNormals) { count++; } diff --git a/libobs-d3d11/d3d11-vertexbuffer.cpp b/libobs-d3d11/d3d11-vertexbuffer.cpp index 585e26db556e8f..f399ee5b10ed8e 100644 --- a/libobs-d3d11/d3d11-vertexbuffer.cpp +++ b/libobs-d3d11/d3d11-vertexbuffer.cpp @@ -48,8 +48,10 @@ void gs_vertex_buffer::FlushBuffer(ID3D11Buffer *buffer, void *array, size_t ele UINT gs_vertex_buffer::MakeBufferList(gs_vertex_shader *shader, ID3D11Buffer **buffers, uint32_t *strides) { UINT numBuffers = 0; - PushBuffer(&numBuffers, buffers, strides, vertexBuffer, sizeof(vec3), "point"); + if (shader->hasPositions) { + PushBuffer(&numBuffers, buffers, strides, vertexBuffer, sizeof(vec3), "point"); + } if (shader->hasNormals) { PushBuffer(&numBuffers, buffers, strides, normalBuffer, sizeof(vec3), "normal"); } From 756150182b346f6b420b0aa8e5a48f075cc13255 Mon Sep 17 00:00:00 2001 From: Richard Stanway Date: Thu, 30 Jul 2026 22:03:25 +0200 Subject: [PATCH 2/2] libobs,frontend: Load vertex and index buffers before drawing We load vertex and index buffers inconsistently between libobs and the frontend - some draws load a vertex buffer without an index buffer and some draws load neither. This causes the device to use whatever the previous draw call left behind, which is not a good practice. While nothing in OBS itself binds an index buffer today, the graphics API is exposed to plugins and any source render callback could leave one bound. --- frontend/utility/display-helpers.hpp | 1 + frontend/widgets/OBSBasicPreview.cpp | 3 +++ frontend/widgets/OBSBasic_Preview.cpp | 1 + libobs/obs-source.c | 2 ++ libobs/obs-video.c | 3 +++ plugins/nv-filters/nvidia-videofx-filter.c | 3 +++ 6 files changed, 13 insertions(+) diff --git a/frontend/utility/display-helpers.hpp b/frontend/utility/display-helpers.hpp index d03f1bb0f0e934..e6c191da43fb45 100644 --- a/frontend/utility/display-helpers.hpp +++ b/frontend/utility/display-helpers.hpp @@ -130,6 +130,7 @@ static inline void RenderSafeAreas(gs_vertbuffer_t *vb, int cx, int cy) transform.y.y = cy; gs_load_vertexbuffer(vb); + gs_load_indexbuffer(nullptr); gs_matrix_push(); gs_matrix_mul(&transform); diff --git a/frontend/widgets/OBSBasicPreview.cpp b/frontend/widgets/OBSBasicPreview.cpp index 4fcd3f5423c958..c41c1a46d6f553 100644 --- a/frontend/widgets/OBSBasicPreview.cpp +++ b/frontend/widgets/OBSBasicPreview.cpp @@ -1784,6 +1784,7 @@ static void DrawRotationHandle(gs_vertbuffer_t *circle, float rot, float pixelRa gs_matrix_translate3f(0.0f, -HANDLE_RADIUS * 2 / 3, 0.0f); gs_load_vertexbuffer(circle); + gs_load_indexbuffer(nullptr); gs_draw(GS_TRISTRIP, 0, 0); gs_matrix_pop(); @@ -2099,6 +2100,7 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *, obs_sceneitem_t *item, voi gs_technique_begin_pass(tech, 0); gs_load_vertexbuffer(main->box); + gs_load_indexbuffer(nullptr); gs_effect_set_vec4(colParam, &red); if (selected) { @@ -2170,6 +2172,7 @@ bool OBSBasicPreview::DrawSelectionBox(float x1, float y1, float x2, float y2, g gs_effect_set_vec4(colParam, &fillColor); gs_load_vertexbuffer(rectFill); + gs_load_indexbuffer(nullptr); gs_draw(GS_TRISTRIP, 0, 0); gs_effect_set_vec4(colParam, &borderColor); diff --git a/frontend/widgets/OBSBasic_Preview.cpp b/frontend/widgets/OBSBasic_Preview.cpp index 79e458d8d77329..7814f1ab89edb6 100644 --- a/frontend/widgets/OBSBasic_Preview.cpp +++ b/frontend/widgets/OBSBasic_Preview.cpp @@ -118,6 +118,7 @@ void OBSBasic::DrawBackdrop(float cx, float cy) gs_matrix_scale3f(float(cx), float(cy), 1.0f); gs_load_vertexbuffer(box); + gs_load_indexbuffer(nullptr); gs_draw(GS_TRISTRIP, 0, 0); gs_matrix_pop(); diff --git a/libobs/obs-source.c b/libobs/obs-source.c index 2e99d9f6d080e6..f691af9e325e66 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -2421,6 +2421,8 @@ static bool update_async_texrender(struct obs_source *source, const struct obs_s gs_effect_set_val(max_param, frame->color_range_max, sizeof(float) * 3); } + gs_load_vertexbuffer(NULL); + gs_load_indexbuffer(NULL); gs_draw(GS_TRIS, 0, 3); gs_technique_end_pass(tech); diff --git a/libobs/obs-video.c b/libobs/obs-video.c index eca6f830402a4c..4e98870d75117c 100644 --- a/libobs/obs-video.c +++ b/libobs/obs-video.c @@ -331,6 +331,9 @@ static void render_convert_plane(gs_effect_t *effect, gs_texture_t *target, cons gs_set_render_target(target, NULL); set_render_size(width, height); + gs_load_vertexbuffer(NULL); + gs_load_indexbuffer(NULL); + size_t passes = gs_technique_begin(tech); for (size_t i = 0; i < passes; i++) { gs_technique_begin_pass(tech, i); diff --git a/plugins/nv-filters/nvidia-videofx-filter.c b/plugins/nv-filters/nvidia-videofx-filter.c index 15341927fe7383..ba254e7ff7aa35 100644 --- a/plugins/nv-filters/nvidia-videofx-filter.c +++ b/plugins/nv-filters/nvidia-videofx-filter.c @@ -944,6 +944,9 @@ static void nvvfx_filter_render(void *data, gs_effect_t *effect, bool has_blur) gs_effect_set_texture_srgb(filter->image_param, gs_texrender_get_texture(render)); gs_effect_set_float(filter->multiplier_param, multiplier); + gs_load_vertexbuffer(NULL); + gs_load_indexbuffer(NULL); + while (gs_effect_loop(filter->effect, tech_name)) { gs_draw(GS_TRIS, 0, 3); }