[3.15] gh-158140: Resolve lazily imported sibling submodules independently (GH-158521) (#158613)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
Co-authored-by: Petr Viktorin <encukou@gmail.com>
Co-authored-by: T. Wouters <thomas@python.org>
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>

Author: 5 people

Include/internal/pycore_import.h

@@ -41,7 +41,9 @@ PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate(
 // new reference, NULL without an exception if absent, or NULL on error.
 // Set recheck_dict if a concurrent lookup may have already bound the child.
 extern PyObject * _PyImport_TryLoadLazySubmodule(
-    PyObject *module, PyObject *attr_name, int *recheck_dict);
+    PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict);
+extern int _PyImport_ClearLazySubmodule(
+    PyThreadState *tstate, PyObject *name, int bind);
 extern PyObject * _PyImport_LazyImportModuleLevelObject(
     PyThreadState *tstate, PyObject *name, PyObject *builtins,
     PyObject *globals, PyObject *locals, PyObject *fromlist, int level);

Include/internal/pycore_interp_structs.h

@@ -352,9 +352,11 @@ struct _import_state {
     // lazily imported. This value is only for debugging/introspection
     // purposes and is not used by the runtime.
     PyObject *lazy_modules;
-    // A dict mapping package names to a set of submodule names that
-    // may need to be imported when an attribute is missing on the package.
+    // Package names map to pending children: declarations for plain imports,
+    // or None for from-import names that may be ordinary attributes.
     PyObject *lazy_pending_submodules;
+    // Avoid pending-child work for ordinary cached imports.
+    int has_lazy_submodules;
 #ifdef Py_GIL_DISABLED
     PyMutex lazy_mutex;
 #endif

Include/internal/pycore_lazyimportobject.h

@@ -18,6 +18,10 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New(
     struct _PyInterpreterFrame *frame, PyObject *builtins,
     PyObject *name, PyObject *fromlist);
 
+extern PyObject *_PyLazyImport_LoadChild(
+    PyThreadState *tstate, PyObject *declaration, PyObject *name);
+extern int _PyLazyImport_IsActive(PyObject *declaration);
+
 extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op);
 
 // Resolve a placeholder and replace its binding if it is unchanged or holds

Lib/test/test_lazy_import/__init__.py

@@ -457,6 +457,121 @@ def test_lazy_with_from(self):
         self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules)
 
 
+@support.requires_subprocess()
+class IndependentSubmoduleTests(LazyImportTestCase):
+    def check(self, code):
+        assert_python_ok('-c', textwrap.dedent(code))
+
+    def test_siblings(self):
+        for package, children in (
+            ('test.test_lazy_import.data.pkg', ('b', 'bar')),
+            ('test.test_lazy_import.data.metasyntactic.foo', ('ack', 'bar')),
+        ):
+            for first, second in (children, children[::-1]):
+                with self.subTest(package=package, first=first):
+                    self.check(f"""
+                        import sys
+                        lazy import {package}.{first}
+                        lazy import {package}.{second}
+                        assert {package}.{first} is sys.modules['{package}.{first}']
+                        assert '{package}.{second}' not in sys.modules
+                        assert {package}.{second} is sys.modules['{package}.{second}']
+                    """)
+
+    def test_star_import(self):
+        self.check("""
+            lazy import urllib.nonexistent
+            lazy import urllib
+            assert urllib.__name__ == 'urllib'
+            from urllib import *
+        """)
+
+    def test_deleted_child(self):
+        self.check("""
+            lazy from xml import dom
+            assert dom.__name__ == 'xml.dom'
+            import xml
+            del xml.dom
+            assert not hasattr(xml, 'dom')
+            lazy from xml import dom
+            assert hasattr(xml, 'dom')
+        """)
+
+    def test_cached_import_releases_builtins(self):
+        for statement in ('lazy from xml.dom import Node',
+                          'lazy import xml.dom.minidom'):
+            with self.subTest(statement=statement):
+                self.check(f"""
+                    import builtins, gc, weakref
+                    import xml.dom.minidom
+                    class Payload:
+                        pass
+                    payload = Payload()
+                    reference = weakref.ref(payload)
+                    namespace = {{'__builtins__': dict(vars(builtins), payload=payload)}}
+                    exec({statement!r}, namespace)
+                    exec('Node' if 'from' in {statement!r} else 'xml.dom.minidom', namespace)
+                    del namespace, payload
+                    gc.collect()
+                    assert reference() is None
+                """)
+
+    def test_invalid_descendant_retries(self):
+        self.check("""
+            lazy import test.test_lazy_import.data.pkg.b.foo
+            for _ in range(2):
+                try:
+                    test.test_lazy_import.data.pkg.b.foo
+                except ModuleNotFoundError:
+                    pass
+                else:
+                    raise AssertionError('imported an attribute as a module')
+            import test.test_lazy_import.data.pkg.b
+            assert callable(test.test_lazy_import.data.pkg.b.foo)
+        """)
+
+    def test_namespace_import_hooks(self):
+        self.check("""
+            import builtins
+            first = {'__builtins__': vars(builtins).copy()}
+            second = {'__builtins__': vars(builtins).copy()}
+            exec('lazy import xml.dom', first)
+            exec('lazy import xml.dom', second)
+            def denied(*args):
+                raise AssertionError('used another namespace import hook')
+            second['__builtins__']['__import__'] = denied
+            exec('assert xml.dom.__name__ == "xml.dom"', first)
+        """)
+
+    def test_retry_after_recursive_access(self):
+        self.check("""
+            import sys, xml
+            from importlib.machinery import ModuleSpec
+            class Loader:
+                calls = 0
+                def find_spec(self, name, path=None, target=None):
+                    if name == 'xml.broken':
+                        return ModuleSpec(name, self)
+                def create_module(self, spec):
+                    return None
+                def exec_module(self, module):
+                    self.calls += 1
+                    xml.broken
+                    raise RuntimeError('failed initialization')
+            loader = Loader()
+            sys.meta_path.insert(0, loader)
+            lazy import xml.broken
+            for _ in range(2):
+                try:
+                    xml.broken
+                except RuntimeError as exc:
+                    assert str(exc) == 'failed initialization'
+                else:
+                    raise AssertionError('returned a failed partial module')
+            assert loader.calls == 2
+        """)
+
+
 class PackageTests(LazyImportTestCase):
     """Tests for lazy imports with packages."""
 
@@ -794,7 +909,7 @@ def test_chained_exception_import_shows_notes(self, name):
             lazy import {name}
 
             try:
-                _ = test
+                _ = {name}
             except ValueError as e:
                 assert any(
                     note.startswith("lazy import of '{name}' declared in ")

Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-20-00-00.gh-issue-158140.small.rst

@@ -0,0 +1,2 @@
+Resolve lazily imported sibling submodules independently, so accessing one
+does not import another merely because its declaration came last.

Objects/lazyimportobject.c

@@ -9,6 +9,7 @@
 #include "pycore_lazyimportobject.h"
 #include "pycore_long.h"
 #include "pycore_moduleobject.h"
+#include "pycore_pyatomic_ft_wrappers.h"
 #include "pycore_pyerrors.h"
 #include "pycore_traceback.h"
 #include "pycore_tstate.h"
@@ -25,6 +26,8 @@ typedef struct {
     // Declaration location.
     PyCodeObject *lz_code;
     int lz_instr_offset;
+    // The root was accessed, so its declared path requires import semantics.
+    int lz_active;
 } PyLazyImportObject;
 
 #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op))
@@ -70,6 +73,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins,
 
     m->lz_code = NULL;
     m->lz_instr_offset = -1;
+    m->lz_active = 0;
 
     if (frame != NULL) {
         m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame));
@@ -429,10 +433,81 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import,
         PyErr_SetString(PyExc_ImportError, "__import__ not found");
         goto done;
     }
+    PyObject *name = Py_NewRef(root->lz_from);
+    if (first == NULL &&
+        (fromlist == Py_None ||
+         (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist) == 0)) &&
+        _PyImport_IsDefaultImportFunc(tstate->interp, import_func) &&
+        root->lz_builtins == tstate->interp->builtins) {
+        // A plain import binds the root package. Its pending children are
+        // imported separately when their attributes are accessed.
+        Py_ssize_t dot = PyUnicode_FindChar(
+            name, '.', 0, PyUnicode_GET_LENGTH(name), 1);
+        if (dot >= 0) {
+            // Preserve eager traversal through aliased or non-package parents.
+            int regular = 1;
+            int complete = 1;
+            Py_ssize_t end = PyUnicode_GET_LENGTH(name);
+            for (Py_ssize_t i = dot; i >= 0 && regular;
+                 i = PyUnicode_FindChar(name, '.', i + 1, end, 1)) {
+                PyObject *prefix = PyUnicode_Substring(name, 0, i);
+                PyObject *cached = prefix == NULL ? NULL :
+                    lazy_import_get_loaded_attr(tstate, prefix, &_Py_ID(__name__));
+                if (cached == NULL) {
+                    complete = 0;
+                    Py_XDECREF(prefix);
+                    break;
+                }
+                PyObject *path = lazy_import_get_loaded_attr(
+                    tstate, prefix, &_Py_ID(__path__));
+                regular = path != NULL && PyUnicode_Check(cached) &&
+                    PyUnicode_Compare(cached, prefix) == 0;
+                Py_XDECREF(path);
+                Py_DECREF(cached);
+                Py_DECREF(prefix);
+            }
+            PyObject *loaded = lazy_import_get_loaded_attr(
+                tstate, name, &_Py_ID(__name__));
+            if (regular && (!complete || loaded == NULL) && !PyErr_Occurred()) {
+                Py_SETREF(name, PyUnicode_Substring(name, 0, dot));
+            }
+            Py_XDECREF(loaded);
+            if (PyErr_Occurred()) {
+                Py_CLEAR(name);
+            }
+        }
+        else if (dot == -2) {
+            Py_CLEAR(name);
+        }
+        if (name == NULL) {
+            goto done;
+        }
+    }
     obj = _PyEval_ImportNameWithImport(
         tstate, import_func, globals, globals,
-        root->lz_from, fromlist, _PyLong_GetZero()
+        name, fromlist, _PyLong_GetZero()
     );
+    if (obj != NULL && name != root->lz_from) {
+        FT_ATOMIC_STORE_INT_RELAXED(root->lz_active, 1);
+        FT_ATOMIC_STORE_INT_RELAXED(tstate->interp->imports.has_lazy_submodules, 1);
+        int package = PyModule_CheckExact(obj) ?
+            PyDict_Contains(_PyModule_GetDict(obj), &_Py_ID(__path__)) : 0;
+        if (package < 0) {
+            Py_CLEAR(obj);
+        }
+        else if (!package) {
+            Py_DECREF(obj);
+            obj = _PyEval_ImportNameWithImport(
+                tstate, import_func, globals, globals,
+                root->lz_from, fromlist, _PyLong_GetZero());
+        }
+    }
+    if (obj != NULL && name == root->lz_from &&
+        _PyImport_IsDefaultImportFunc(tstate->interp, import_func) &&
+        _PyImport_ClearLazySubmodule(tstate, name, 0) < 0) {
+        Py_CLEAR(obj);
+    }
+    Py_DECREF(name);
     obj = lazy_import_resolve_result(tstate, obj);
     // The normal importer may publish this module on its parent. Custom
     // hooks retain control of their own assignments to the parent.
@@ -453,7 +528,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import,
     }
     assert(obj == NULL || !PyLazyImport_CheckExact(obj));
     if (obj != NULL) {
-        PyObject *name = lazy_import_name(lz);
+        PyObject *name = lazy_import_path(lz);
         if (name == NULL ||
             _PyImport_DiscardLazyModule(tstate->interp, name) < 0) {
             Py_CLEAR(obj);
@@ -484,6 +559,58 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
     return lazy_import_resolve_impl(tstate, lazy_import, NULL);
 }
 
+// Resolve a pending child as an aliased import, preserving its declaration.
+PyObject *
+_PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration,
+                        PyObject *name)
+{
+    assert(PyLazyImport_CheckExact(declaration));
+    PyLazyImportObject *source = PyLazyImportObject_CAST(declaration);
+    PyObject *lz = _PyLazyImport_New(NULL, source->lz_builtins, name, NULL);
+    Py_ssize_t end = PyUnicode_GET_LENGTH(name);
+    Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, 1);
+    if (dot == -2) {
+        Py_CLEAR(lz);
+    }
+    while (lz != NULL && dot >= 0) {
+        Py_ssize_t start = dot + 1;
+        dot = PyUnicode_FindChar(name, '.', start, end, 1);
+        if (dot == -2) {
+            Py_CLEAR(lz);
+            break;
+        }
+        PyObject *attr = PyUnicode_Substring(name, start, dot < 0 ? end : dot);
+        PyObject *next = attr == NULL ? NULL :
+            _PyLazyImport_New(NULL, NULL, lz, attr);
+        Py_XDECREF(attr);
+        Py_SETREF(lz, next);
+    }
+    if (lz == NULL) {
+        return NULL;
+    }
+    PyLazyImportObject *child = PyLazyImportObject_CAST(lz);
+    child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code);
+    child->lz_instr_offset = source->lz_instr_offset;
+    PyObject *result = _PyImport_LoadLazyImportTstate(tstate, lz);
+    Py_DECREF(lz);
+    if (result != NULL && PyModule_Check(result) &&
+        PyUnicode_Compare(source->lz_from, name) != 0 &&
+        !PyDict_Contains(_PyModule_GetDict(result), &_Py_ID(__path__))) {
+        // Reaching a non-package must still reject an invalid dotted import.
+        Py_DECREF(result);
+        return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from);
+    }
+    return result;
+}
+
+int
+_PyLazyImport_IsActive(PyObject *declaration)
+{
+    assert(PyLazyImport_CheckExact(declaration));
+    return FT_ATOMIC_LOAD_INT_RELAXED(
+        PyLazyImportObject_CAST(declaration)->lz_active);
+}
+
 // Loading pkg.child can replace a placeholder in pkg.child with the module
 // before a from-import retrieves the value that belongs in that binding.
 // This is an optimization that can be safely skipped.

Objects/moduleobject.c

@@ -1338,7 +1338,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress)
         }
         assert(m->md_dict != NULL);
         int recheck_dict;
-        attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, &recheck_dict);
+        attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, suppress,
+                                             &recheck_dict);
         if (attr != NULL || PyErr_Occurred()) {
             return attr;
         }