From ef976c573cb20eadb22d99312c9ade9d76bc9a23 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 3 Jun 2026 12:32:17 -0700 Subject: [PATCH 1/4] Guard against import machinery masking out attributes --- src/lazy_loader/__init__.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/lazy_loader/__init__.py b/src/lazy_loader/__init__.py index 23b5382..0af4e5d 100644 --- a/src/lazy_loader/__init__.py +++ b/src/lazy_loader/__init__.py @@ -21,6 +21,36 @@ threadlock = threading.Lock() +class _ShadowGuardModule(types.ModuleType): + """Module type to protect function attributes from being overwritten. + + When a function has the same name as the submodule it resides in + (e.g. a ``max_tree`` function defined in ``max_tree.py``), + importing that submodule causes the import machinery to call + ``setattr(pkg, "max_tree", )``. That updates the + package ``__dict__``, preventing ``__getattr__`` from ever + resolving the name to the function again. + + This subclass suppresses those dictionary updates (only in the + shadowing case). + + We track the set of protected names in the ``__lazy_shadowed__`` + attr. + + """ + + def __setattr__(self, name, value): + shadowed = self.__dict__.get("__lazy_shadowed__") + if ( + shadowed is not None + and name in shadowed + # Is it trying to set this attribute to the system module? + and value is sys.modules.get(f"{self.__name__}.{name}") + ): + return + super().__setattr__(name, value) + + def attach(package_name, submodules=None, submod_attrs=None): """Attach lazily loaded submodules, functions, or other attributes. @@ -92,6 +122,26 @@ def __getattr__(name): def __dir__(): return __all__.copy() + # When a function attribute has the same name as the submodule it + # resides in (e.g. `max_tree` from `max_tree.py`), importing that + # submodule makes the import machinery overwrite the parent + # package attribute with the module object, shadowing the function + # (see _ShadowGuardModule). + # + # Record affected cases and, only in those cases, swap in the + # guarding module type. + shadowed = {attr for attr, mod in attr_to_modules.items() if attr == mod} + if shadowed: + pkg = sys.modules.get(package_name) + # Only touch plain package modules (or our own wrapper) --- we + # don't want to mess with custom module classes. + if type(pkg) in (types.ModuleType, _ShadowGuardModule): + pkg.__dict__["__lazy_shadowed__"] = ( + pkg.__dict__.get("__lazy_shadowed__", set()) | shadowed + ) + if type(pkg) is types.ModuleType: + pkg.__class__ = _ShadowGuardModule + eager_import = os.environ.get("EAGER_IMPORT", "") not in ("0", "") if eager_import: for attr in set(attr_to_modules.keys()) | submodules: From e788bb19b00c62579f04c6a1ea36e5447c81e323 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 3 Jun 2026 12:49:52 -0700 Subject: [PATCH 2/4] Add regression test --- tests/test_lazy_loader.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_lazy_loader.py b/tests/test_lazy_loader.py index d68537f..074395e 100644 --- a/tests/test_lazy_loader.py +++ b/tests/test_lazy_loader.py @@ -178,6 +178,17 @@ def test_attach_same_module_and_attr_name(clean_fake_pkg, eager_import): assert isinstance(some_func, types.FunctionType) +def test_attach_submodule_does_not_shadow_function(clean_fake_pkg): + # Where `some_func` is defined in module `some_func`: When + # submodule is imported before the function has been resolved, the + # import machinery tries to set the package `__dict__` to point to + # the module. We need to prevent this, otherwise we cannot + # access the function. + import tests.fake_pkg.some_func # noqa: F401 + from tests import fake_pkg + assert isinstance(fake_pkg.some_func, types.FunctionType) + + FAKE_STUB = """ from . import rank from ._gaussian import gaussian From 746aa7f164be3b33186e8d78eda16b02f281dfc5 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 10 Jun 2026 07:42:11 +0000 Subject: [PATCH 3/4] Also guard against nested subpackage shadowing --- src/lazy_loader/__init__.py | 20 +++++++++++++------- tests/fake_pkg_submodule/__init__.py | 5 +++++ tests/fake_pkg_submodule/x/__init__.py | 0 tests/fake_pkg_submodule/x/sub.py | 2 ++ tests/test_lazy_loader.py | 20 ++++++++++++++++---- 5 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 tests/fake_pkg_submodule/__init__.py create mode 100644 tests/fake_pkg_submodule/x/__init__.py create mode 100644 tests/fake_pkg_submodule/x/sub.py diff --git a/src/lazy_loader/__init__.py b/src/lazy_loader/__init__.py index 0af4e5d..f8034e6 100644 --- a/src/lazy_loader/__init__.py +++ b/src/lazy_loader/__init__.py @@ -29,7 +29,8 @@ class _ShadowGuardModule(types.ModuleType): importing that submodule causes the import machinery to call ``setattr(pkg, "max_tree", )``. That updates the package ``__dict__``, preventing ``__getattr__`` from ever - resolving the name to the function again. + resolving the name to the function again. The same problem occurs + when ``x`` is defined in ``x/sub.py``. This subclass suppresses those dictionary updates (only in the shadowing case). @@ -122,15 +123,20 @@ def __getattr__(name): def __dir__(): return __all__.copy() - # When a function attribute has the same name as the submodule it - # resides in (e.g. `max_tree` from `max_tree.py`), importing that - # submodule makes the import machinery overwrite the parent - # package attribute with the module object, shadowing the function - # (see _ShadowGuardModule). + # When a function has the same name as a module the import + # machinery needs to load along the way to accessing it + # (e.g. `max_tree` from `max_tree.py`, or `x` from `x/sub.py`), a + # side-effect of it loading that module is overwriting the package + # attribute (so it points to the module, i.e. to `max_tree` or `x` + # the module), shadowing the function (see _ShadowGuardModule). # # Record affected cases and, only in those cases, swap in the # guarding module type. - shadowed = {attr for attr, mod in attr_to_modules.items() if attr == mod} + shadowed = { + attr + for attr, mod in attr_to_modules.items() + if attr == mod.split(".")[0] + } if shadowed: pkg = sys.modules.get(package_name) # Only touch plain package modules (or our own wrapper) --- we diff --git a/tests/fake_pkg_submodule/__init__.py b/tests/fake_pkg_submodule/__init__.py new file mode 100644 index 0000000..c6d6174 --- /dev/null +++ b/tests/fake_pkg_submodule/__init__.py @@ -0,0 +1,5 @@ +import lazy_loader as lazy + +__getattr__, __dir__, __all__ = lazy.attach( + __name__, submod_attrs={"x.sub": ["x"]} +) diff --git a/tests/fake_pkg_submodule/x/__init__.py b/tests/fake_pkg_submodule/x/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fake_pkg_submodule/x/sub.py b/tests/fake_pkg_submodule/x/sub.py new file mode 100644 index 0000000..127bbcc --- /dev/null +++ b/tests/fake_pkg_submodule/x/sub.py @@ -0,0 +1,2 @@ +def x(): + """Function with the same name as the package it lives under.""" diff --git a/tests/test_lazy_loader.py b/tests/test_lazy_loader.py index 074395e..64c5109 100644 --- a/tests/test_lazy_loader.py +++ b/tests/test_lazy_loader.py @@ -13,9 +13,9 @@ @pytest.fixture def clean_fake_pkg(): yield - sys.modules.pop("tests.fake_pkg.some_func", None) - sys.modules.pop("tests.fake_pkg", None) - sys.modules.pop("tests", None) + for name in [*sys.modules]: + if name == "tests" or name.startswith("tests.fake_pkg"): + del sys.modules[name] @pytest.mark.parametrize("attempt", [1, 2]) @@ -179,7 +179,7 @@ def test_attach_same_module_and_attr_name(clean_fake_pkg, eager_import): def test_attach_submodule_does_not_shadow_function(clean_fake_pkg): - # Where `some_func` is defined in module `some_func`: When + # Where `some_func` is defined in module `some_func`: when # submodule is imported before the function has been resolved, the # import machinery tries to set the package `__dict__` to point to # the module. We need to prevent this, otherwise we cannot @@ -189,6 +189,18 @@ def test_attach_submodule_does_not_shadow_function(clean_fake_pkg): assert isinstance(fake_pkg.some_func, types.FunctionType) +def test_attach_subpackage_does_not_shadow_function(clean_fake_pkg): + # Where function `x` is defined in `x/sub.py`, i.e. a module + # nested inside a subpackage that shares its name with the + # function: importing `x.sub` causes the import machinery to set + # the package attribute `x` to the `x` subpackage, which would + # otherwise shadow the `x` function on subsequent access. + from tests import fake_pkg_submodule + + assert isinstance(fake_pkg_submodule.x, types.FunctionType) + assert isinstance(fake_pkg_submodule.x, types.FunctionType) + + FAKE_STUB = """ from . import rank from ._gaussian import gaussian From bd25670b2c797805fe480ca603fb747e302f8b71 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 10 Jun 2026 00:44:36 -0700 Subject: [PATCH 4/4] Satisfy linter --- src/lazy_loader/__init__.py | 4 +--- tests/fake_pkg_submodule/__init__.py | 4 +--- tests/test_lazy_loader.py | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lazy_loader/__init__.py b/src/lazy_loader/__init__.py index f8034e6..b0dc2e1 100644 --- a/src/lazy_loader/__init__.py +++ b/src/lazy_loader/__init__.py @@ -133,9 +133,7 @@ def __dir__(): # Record affected cases and, only in those cases, swap in the # guarding module type. shadowed = { - attr - for attr, mod in attr_to_modules.items() - if attr == mod.split(".")[0] + attr for attr, mod in attr_to_modules.items() if attr == mod.split(".")[0] } if shadowed: pkg = sys.modules.get(package_name) diff --git a/tests/fake_pkg_submodule/__init__.py b/tests/fake_pkg_submodule/__init__.py index c6d6174..daa6896 100644 --- a/tests/fake_pkg_submodule/__init__.py +++ b/tests/fake_pkg_submodule/__init__.py @@ -1,5 +1,3 @@ import lazy_loader as lazy -__getattr__, __dir__, __all__ = lazy.attach( - __name__, submod_attrs={"x.sub": ["x"]} -) +__getattr__, __dir__, __all__ = lazy.attach(__name__, submod_attrs={"x.sub": ["x"]}) diff --git a/tests/test_lazy_loader.py b/tests/test_lazy_loader.py index 64c5109..0f7337a 100644 --- a/tests/test_lazy_loader.py +++ b/tests/test_lazy_loader.py @@ -186,6 +186,7 @@ def test_attach_submodule_does_not_shadow_function(clean_fake_pkg): # access the function. import tests.fake_pkg.some_func # noqa: F401 from tests import fake_pkg + assert isinstance(fake_pkg.some_func, types.FunctionType)