diff --git a/lib/ex_unit/lib/ex_unit/capture_server.ex b/lib/ex_unit/lib/ex_unit/capture_server.ex
index 5eb4f4a18b4..a9307f1bcb8 100644
--- a/lib/ex_unit/lib/ex_unit/capture_server.ex
+++ b/lib/ex_unit/lib/ex_unit/capture_server.ex
@@ -155,6 +155,7 @@ defmodule ExUnit.CaptureServer do
       Process.register(pid, name)
     rescue
       ArgumentError ->
+        close_string_io(pid)
         {:reply, {:error, :no_device}, config}
     else
       _ ->
diff --git a/lib/ex_unit/test/ex_unit/capture_io_test.exs b/lib/ex_unit/test/ex_unit/capture_io_test.exs
index bd0ea681167..97cd5762f2a 100644
--- a/lib/ex_unit/test/ex_unit/capture_io_test.exs
+++ b/lib/ex_unit/test/ex_unit/capture_io_test.exs
@@ -477,6 +477,17 @@ defmodule ExUnit.CaptureIOTest do
       end
     end
 
+    test "does not leak StringIO when named device does not exist" do
+      capture_server = Process.whereis(ExUnit.CaptureServer)
+      {:monitored_by, monitored_by} = Process.info(capture_server, :monitored_by)
+
+      assert_raise RuntimeError, "could not find IO device registered at :unknown_device", fn ->
+        capture_io(:unknown_device, fn -> :ok end)
+      end
+
+      assert Process.info(capture_server, :monitored_by) == {:monitored_by, monitored_by}
+    end
+
     test "no leakage on failures" do
       parent = self()