Use mutex on GIL-enabled builds too.

This race is not free-threading specific.

Author: nascheme

Lib/test/test_free_threading/test_os.py

@@ -24,6 +24,7 @@
 import sysconfig
 import tempfile
 import textwrap
+import threading
 import time
 import types
 import unittest
@@ -35,6 +36,7 @@
 from test.support import infinite_recursion
 from test.support import requires_root_user
 from test.support import requires_non_root_user
+from test.support import threading_helper
 from test.support import warnings_helper
 from platform import win32_is_iot
 from .utils import create_file
@@ -5351,6 +5353,72 @@ def test_resource_warning(self):
             del iterator
 
 
+@threading_helper.requires_working_threading()
+class ScandirThreadingTest(unittest.TestCase):
+    # gh-152754: an os.scandir() iterator shared between threads must not crash.
+
+    if support.check_sanitizer(thread=True):
+        SCANDIR_NUMITEMS = 200
+        SCANDIR_N_NEXT = 2
+        SCANDIR_N_CLOSE = 2
+        SCANDIR_REPEAT = 10
+    else:
+        SCANDIR_NUMITEMS = 1000
+        SCANDIR_N_NEXT = 6
+        SCANDIR_N_CLOSE = 3
+        SCANDIR_REPEAT = 20
+
+    def setUp(self):
+        self.dir = os.path.realpath(os_helper.TESTFN)
+        self.addCleanup(os_helper.rmtree, self.dir)
+        os.mkdir(self.dir)
+        self.names = set()
+        for i in range(self.SCANDIR_NUMITEMS):
+            name = f"f{i}"
+            create_file(os.path.join(self.dir, name))
+            self.names.add(name)
+
+    def test_close_racing_next(self):
+        # One thread's next() racing another's close() must not crash.
+        def nexter():
+            for _ in self.it:
+                pass
+
+        def closer():
+            self.it.close()
+
+        funcs = [nexter] * self.SCANDIR_N_NEXT + [closer] * self.SCANDIR_N_CLOSE
+        for _ in range(self.SCANDIR_REPEAT):
+            self.it = os.scandir(self.dir)
+            try:
+                threading_helper.run_concurrently(funcs)
+            finally:
+                self.it.close()
+
+    def test_shared_next(self):
+        # Threads sharing one iterator must not crash or lose entries: every
+        # entry must be handed to exactly one thread.
+        expected = sorted(self.names)
+        nthreads = self.SCANDIR_N_NEXT + self.SCANDIR_N_CLOSE
+
+        for _ in range(self.SCANDIR_REPEAT):
+            self.it = os.scandir(self.dir)
+            results = []
+            results_lock = threading.Lock()
+
+            def worker():
+                local = [entry.name for entry in self.it]
+                with results_lock:
+                    results.extend(local)
+
+            try:
+                threading_helper.run_concurrently([worker] * nthreads)
+            finally:
+                self.it.close()
+
+            self.assertEqual(sorted(results), expected)
+
+
 class TestPEP519(unittest.TestCase):
 
     # Abstracted so it can be overridden to test pure Python implementation

Lib/test/test_os/test_os.py

@@ -1,2 +1,4 @@
 Fix a crash when the same :func:`os.scandir` iterator is used concurrently
-from multiple threads on the :term:`free-threaded <free threading>` build.
+from multiple threads.  Iterating and closing the iterator are now serialized
+with a per-iterator lock, so that closing it no longer releases the
+directory handle while another thread is still reading from it.

Misc/NEWS.d/next/Library/2026-07-09-14-56-24.gh-issue-152754.CyyC5j.rst

@@ -24,9 +24,9 @@
 #include "pycore_long.h"          // _PyLong_IsNegative()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
 #include "pycore_object.h"        // _PyObject_LookupSpecial()
+#include "pycore_pyatomic_ft_wrappers.h"  // FT_ATOMIC_LOAD_INT_RELAXED()
 #include "pycore_pylifecycle.h"   // _PyOS_URandom()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_pyatomic_ft_wrappers.h" // FT_MUTEX_LOCK()
 #include "pycore_signal.h"        // Py_NSIG
 #include "pycore_time.h"          // _PyLong_FromTime_t()
 #include "pycore_tuple.h"         // _PyTuple_FromPairSteal
@@ -16941,11 +16941,6 @@ DirEntry_from_posix_info(PyObject *module, path_t *path, const char *name,
 
 typedef struct {
     PyObject_HEAD
-#ifdef Py_GIL_DISABLED
-    // Protects scandir iterator state when a os.scandir() iterator is used
-    // from multiple threads.
-    PyMutex mutex;
-#endif
     path_t path;
 #ifdef MS_WINDOWS
     HANDLE handle;
@@ -16957,6 +16952,9 @@ typedef struct {
 #ifdef HAVE_FDOPENDIR
     int fd;
 #endif
+    // Protects the iterator state when an os.scandir() iterator is used from
+    // multiple threads.
+    PyMutex mutex;
 } ScandirIterator;
 
 #define ScandirIterator_CAST(op)    ((ScandirIterator *)(op))
@@ -16966,19 +16964,19 @@ typedef struct {
 static int
 ScandirIterator_is_closed(ScandirIterator *iterator)
 {
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     int closed = iterator->handle == INVALID_HANDLE_VALUE;
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
     return closed;
 }
 
 static void
 ScandirIterator_closedir(ScandirIterator *iterator)
 {
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     HANDLE handle = iterator->handle;
     iterator->handle = INVALID_HANDLE_VALUE;
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
 
     if (handle != INVALID_HANDLE_VALUE) {
         Py_BEGIN_ALLOW_THREADS
@@ -16996,7 +16994,7 @@ ScandirIterator_iternext(PyObject *op)
     DWORD error = ERROR_SUCCESS;
     int found = 0;
 
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     /* Happens if the iterator is iterated twice, or closed explicitly */
     while (iterator->handle != INVALID_HANDLE_VALUE) {
         if (!iterator->first_time) {
@@ -17023,7 +17021,7 @@ ScandirIterator_iternext(PyObject *op)
 
         /* Loop till we get a non-dot directory or finish iterating */
     }
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
 
     if (found) {
         PyObject *module = PyType_GetModule(Py_TYPE(iterator));
@@ -17047,19 +17045,19 @@ ScandirIterator_iternext(PyObject *op)
 static int
 ScandirIterator_is_closed(ScandirIterator *iterator)
 {
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     int closed = iterator->dirp == NULL;
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
     return closed;
 }
 
 static void
 ScandirIterator_closedir(ScandirIterator *iterator)
 {
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     DIR *dirp = iterator->dirp;
     iterator->dirp = NULL;
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
 
     if (dirp != NULL) {
         Py_BEGIN_ALLOW_THREADS
@@ -17089,7 +17087,7 @@ ScandirIterator_iternext(PyObject *op)
     unsigned char d_type = 0;
 #endif
 
-    FT_MUTEX_LOCK(&iterator->mutex);
+    PyMutex_Lock(&iterator->mutex);
     /* Happens if the iterator is iterated twice, or closed explicitly */
     while (iterator->dirp != NULL) {
         Py_BEGIN_ALLOW_THREADS
@@ -17126,7 +17124,7 @@ ScandirIterator_iternext(PyObject *op)
 
         /* Loop till we get a non-dot directory or finish iterating */
     }
-    FT_MUTEX_UNLOCK(&iterator->mutex);
+    PyMutex_Unlock(&iterator->mutex);
 
     if (found) {
         PyObject *module = PyType_GetModule(Py_TYPE(iterator));
@@ -17286,9 +17284,7 @@ os_scandir_impl(PyObject *module, path_t *path)
     if (!iterator)
         return NULL;
 
-#ifdef Py_GIL_DISABLED
     iterator->mutex = (PyMutex){0};
-#endif
 #ifdef MS_WINDOWS
     iterator->handle = INVALID_HANDLE_VALUE;
 #else