Fix crash when a destructor uses lazy imports during shutdown

Destructors can run after finalization has cleared the interpreter's
lazy_modules set and lazy_pending_submodules dict. Declaring or
resolving a lazy import from one of them passed NULL to PySet_Add(),
PySet_Discard() or the pending-submodules lookup.

Treat the bookkeeping as a no-op once the state is gone, as the other
readers of this state already do.

Author: brittanyrey

Lib/test/test_lazy_import/__init__.py

@@ -2807,5 +2807,45 @@ def test_fails_without_lazy(self):
         self.assertIn("ImportError", result.stderr)
 
 
+@support.requires_subprocess()
+class LazyImportFinalizationTests(unittest.TestCase):
+    """Destructors that use lazy imports after finalization freed their state."""
+
+    def test_declare(self):
+        code = textwrap.dedent("""
+            import builtins, os
+
+            class Canary:
+                def __del__(self, write=os.write, exec=exec,
+                            builtins={"__lazy_import__": builtins.__lazy_import__}):
+                    exec("lazy import a.b", {"__builtins__": builtins})
+                    write(1, b"ok")
+
+            # Only this placeholder keeps the canary alive.
+            exec("lazy import pkg.mod", {"__builtins__": {
+                "__lazy_import__": builtins.__lazy_import__, "c": Canary()}})
+        """)
+        self.assertEqual(assert_python_ok("-c", code).out, b"ok")
+
+    def test_resolve(self):
+        code = textwrap.dedent("""
+            import contextvars, json, os, types
+
+            def safe_import(name, *args, allowed={"json": json}):
+                return allowed[name]
+
+            # Not in sys.modules, with builtins that still work at shutdown.
+            ns = types.ModuleType("ns")
+            ns.json = __lazy_import__("json", {"__builtins__": {"__import__": safe_import}})
+
+            class Canary:
+                def __del__(self, write=os.write, type=type, ns=ns):
+                    write(1, type(ns.json).__name__.encode())
+
+            contextvars.ContextVar("v").set(Canary())
+        """)
+        self.assertEqual(assert_python_ok("-c", code).out, b"module")
+
+
 if __name__ == '__main__':
     unittest.main()

Python/import.c

@@ -287,6 +287,10 @@ _PyImport_ClearLazyModules(PyInterpreterState *interp)
 int
 _PyImport_DiscardLazyModule(PyInterpreterState *interp, PyObject *name)
 {
+    // Destructors can still run after finalization cleared the set.
+    if (LAZY_MODULES(interp) == NULL) {
+        return 0;
+    }
     return PySet_Discard(LAZY_MODULES(interp), name);
 }
 
@@ -4179,7 +4183,10 @@ lazy_modules_add(PyThreadState *tstate, PyObject *name,
     else {
         Py_XDECREF(mod);
     }
-    return loaded ? 0 : PySet_Add(LAZY_MODULES(tstate->interp), name);
+    if (loaded || LAZY_MODULES(tstate->interp) == NULL) {
+        return 0;
+    }
+    return PySet_Add(LAZY_MODULES(tstate->interp), name);
 }
 
 // Ensure a dict of pending submodule names exists for the parent.
@@ -4213,7 +4220,10 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source)
 {
     PyDictObject *pending =
         (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp);
-    assert(pending != NULL);
+    // Destructors can still run after finalization cleared the dict.
+    if (pending == NULL) {
+        return 0;
+    }
     Py_ssize_t end = PyUnicode_GET_LENGTH(name);
     while (true) {
         Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1);
@@ -5575,7 +5585,7 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj,
 /*[clinic end generated code: output=3369bb3242b1f043 input=900339e013ab2b82]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    if (PySet_Discard(LAZY_MODULES(interp), name) < 0) {
+    if (_PyImport_DiscardLazyModule(interp, name) < 0) {
         return NULL;
     }
     if (_PyImport_ClearLazySubmodule(_PyThreadState_GET(), name, 0) < 0) {