gh-142349: Fix refcount corruption in lazy import specialization

The LOAD_GLOBAL specialization path in specialize_load_global_lock_held()
called Py_DECREF(value) on the result of _PyDict_LookupIndexAndValue(),
which returns a borrowed reference via _Py_dict_lookup(). This spurious
DECREF silently corrupted the reference count of lazy import objects each
time the adaptive specialization counter triggered while the lazy import
was still present in the globals dict. This happens during concurrent lazy
import reification with 8 or more threads in practice.

The reason the thread count mattered is that with more threads racing
through the barrier, more of them got a turn on the GIL to execute the
LOAD_GLOBAL instruction (and trigger its adaptive specialization
counter) while the first thread was still inside the import machinery
resolving the lazy import. Each such trigger stole one reference from
the lazy import object. With 2-7 threads, typically only 1-2
specialization triggers fired before the import completed, so the object
survived. With 8+ threads, enough triggers accumulated (3 or more) to
drive the reference count to zero while other threads still held
pointers to the object, causing use-after-free.

The fix removes the erroneous Py_DECREF from the specialization guard.
The module attribute specialization path (specialize_module_load_attr)
correctly handled the same case without a DECREF and served as
confirmation that the LOAD_GLOBAL path was wrong. This commit also adds
the lz_resolved field to PyLazyImportObject, which caches the resolved
module after the first thread completes the import. Subsequent threads
that were blocked on the import lock can then return the cached result
immediately rather than re-importing. The _PyImport_LoadLazyImportTstate
function now takes a strong reference to the lazy import at entry (to
keep it alive across the GIL release inside _PyImport_AcquireLock) and
releases it at every exit path, and a double-release of the import lock
in the PySet_Add error path has also been corrected.

Author: pablogsal

Include/internal/pycore_lazyimportobject.h

@@ -22,6 +22,10 @@ typedef struct {
     // Frame information for the original import location.
     PyCodeObject *lz_code;     // Code object where the lazy import was created.
     int lz_instr_offset;       // Instruction offset where the lazy import was created.
+    // Cached result from the first thread to resolve this lazy import.
+    // Protected by the global import lock. When non-NULL, subsequent
+    // threads skip the import and return this value directly.
+    PyObject *lz_resolved;
 } PyLazyImportObject;
 
 

Objects/lazyimportobject.c

@@ -37,6 +37,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name
     // Capture frame information for the original import location.
     m->lz_code = NULL;
     m->lz_instr_offset = -1;
+    m->lz_resolved = NULL;
 
     if (frame != NULL) {
         PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -59,6 +60,7 @@ lazy_import_traverse(PyObject *op, visitproc visit, void *arg)
     Py_VISIT(m->lz_from);
     Py_VISIT(m->lz_attr);
     Py_VISIT(m->lz_code);
+    Py_VISIT(m->lz_resolved);
     return 0;
 }
 
@@ -70,6 +72,7 @@ lazy_import_clear(PyObject *op)
     Py_CLEAR(m->lz_from);
     Py_CLEAR(m->lz_attr);
     Py_CLEAR(m->lz_code);
+    Py_CLEAR(m->lz_resolved);
     return 0;
 }
 

Python/import.c

@@ -1,6 +1,7 @@
 /* Module definition and import implementation */
 
 #include "Python.h"
+
 #include "pycore_audit.h"         // _PySys_Audit()
 #include "pycore_ceval.h"
 #include "pycore_critical_section.h"  // Py_BEGIN_CRITICAL_SECTION()
@@ -3881,12 +3882,29 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
     assert(lazy_import != NULL);
     assert(PyLazyImport_CheckExact(lazy_import));
 
+    // Keep a strong reference to the lazy import object. During
+    // _PyImport_AcquireLock() below the GIL may be released, allowing
+    // another thread to resolve this same lazy import and drop all
+    // external references (from the globals dict and stack). The extra
+    // INCREF ensures the object stays alive while we work with it.
+    Py_INCREF(lazy_import);
+
     PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import;
     PyInterpreterState *interp = tstate->interp;
 
     // Acquire the global import lock to serialize reification
     _PyImport_AcquireLock(interp);
 
+    // Check if another thread already resolved this lazy import while
+    // we were waiting for the lock. If so, return the cached result.
+    PyObject *resolved = FT_ATOMIC_LOAD_PTR_ACQUIRE(lz->lz_resolved);
+    if (resolved != NULL) {
+        obj = Py_NewRef(resolved);
+        _PyImport_ReleaseLock(interp);
+        Py_DECREF(lazy_import);
+        return obj;
+    }
+
     // Check if we are already importing this module, if so, then we want to
     // return an error that indicates we've hit a cycle which will indicate
     // the value isn't yet available.
@@ -3895,20 +3913,24 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
         importing = interp->imports.lazy_importing_modules = PySet_New(NULL);
         if (importing == NULL) {
             _PyImport_ReleaseLock(interp);
+            Py_DECREF(lazy_import);
             return NULL;
         }
     }
 
     assert(PyAnySet_CheckExact(importing));
+
     int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import);
     if (is_loading < 0) {
         _PyImport_ReleaseLock(interp);
+        Py_DECREF(lazy_import);
         return NULL;
     }
     else if (is_loading == 1) {
         PyObject *name = _PyLazyImport_GetName(lazy_import);
         if (name == NULL) {
             _PyImport_ReleaseLock(interp);
+            Py_DECREF(lazy_import);
             return NULL;
         }
         PyObject *errmsg = PyUnicode_FromFormat(
@@ -3917,17 +3939,18 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
         if (errmsg == NULL) {
             Py_DECREF(name);
             _PyImport_ReleaseLock(interp);
+            Py_DECREF(lazy_import);
             return NULL;
         }
         PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg,
                                      lz->lz_from, NULL);
         Py_DECREF(errmsg);
         Py_DECREF(name);
         _PyImport_ReleaseLock(interp);
+        Py_DECREF(lazy_import);
         return NULL;
     }
     else if (PySet_Add(importing, lazy_import) < 0) {
-        _PyImport_ReleaseLock(interp);
         goto error;
     }
 
@@ -3986,6 +4009,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
         );
         Py_DECREF(name);
     }
+
     if (obj == NULL) {
         goto error;
     }
@@ -4001,6 +4025,10 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
 
     assert(!PyLazyImport_CheckExact(obj));
 
+    // Cache the resolved result so other threads waiting on the import
+    // lock can return it immediately without re-importing.
+    FT_ATOMIC_STORE_PTR_RELEASE(lz->lz_resolved, Py_NewRef(obj));
+
     goto ok;
 
 error:
@@ -4090,6 +4118,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
 
     Py_XDECREF(fromlist);
     Py_XDECREF(import_func);
+    Py_DECREF(lazy_import);
     return obj;
 }
 

Python/specialize.c

@@ -1321,7 +1321,6 @@ specialize_load_global_lock_held(
     }
     if (value != NULL && PyLazyImport_CheckExact(value)) {
         SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_ATTR_MODULE_LAZY_VALUE);
-        Py_DECREF(value);
         goto fail;
     }
     PyInterpreterState *interp = _PyInterpreterState_GET();