Ensure namespaces from ImportFinder handle additions to path

According to the PEP 420, namespace packages need to gracefully handle
later additions to path.

- Use a `PathEntryFinder` + an arbitrary placeholder entry on `sys.path`
  to force `PathFinder` to create a namespace spec.
  - Since `_NamespacePath` and `_NamespaceLoader` are private classes (or
    just documented for comparison purposes), there is no other way to
    implement this behaviour directly [^1].

[^1]: Reimplementing _NamespacePath + a custom logic to maintain
      namespace portions don't have a corresponding path entry also
      seems to have the same end result.

Author: abravalheri

setuptools/command/editable_wheel.py

@@ -18,7 +18,7 @@
 from itertools import chain
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union, TypeVar
+from typing import Dict, Iterable, Iterator, List, Mapping, Union, Tuple, TypeVar
 
 from setuptools import Command, namespaces
 from setuptools.discovery import find_package_path
@@ -247,10 +247,15 @@ def __call__(self, unpacked_wheel_dir: Path):
         top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist))
         package_dir = self.dist.package_dir or {}
         roots = _find_package_roots(top_level, package_dir, src_root)
-        namespaces_ = set(_find_mapped_namespaces(roots))
 
-        finder = _make_identifier(f"__editable__.{self.name}.finder")
-        content = _finder_template(roots, namespaces_)
+        namespaces_: Dict[str, List[str]] = dict(chain(
+            _find_namespaces(self.dist.packages, roots),
+            ((ns, []) for ns in _find_virtual_namespaces(roots)),
+        ))
+
+        name = f"__editable__.{self.name}.finder"
+        finder = _make_identifier(name)
+        content = _finder_template(name, roots, namespaces_)
         Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8")
 
         pth = f"__editable__.{self.name}.pth"
@@ -398,9 +403,9 @@ def _absolute_root(path: _Path) -> str:
         return str(parent.resolve() / path_.name)
 
 
-def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
-    """By carefully designing ``package_dir``, it is possible to implement
-    PEP 420 compatible namespaces without creating extra folders.
+def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
+    """By carefully designing ``package_dir``, it is possible to implement the logical
+    structure of PEP 420 in a package without the corresponding directories.
     This function will try to find this kind of namespaces.
     """
     for pkg in pkg_roots:
@@ -409,11 +414,20 @@ def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]:
         parts = pkg.split(".")
         for i in range(len(parts) - 1, 0, -1):
             partial_name = ".".join(parts[:i])
-            path = find_package_path(partial_name, pkg_roots, "")
-            if not Path(path, "__init__.py").exists():
+            path = Path(find_package_path(partial_name, pkg_roots, ""))
+            if not path.exists():
                 yield partial_name
 
 
+def _find_namespaces(
+    packages: List[str], pkg_roots: Dict[str, str]
+) -> Iterator[Tuple[str, List[str]]]:
+    for pkg in packages:
+        path = find_package_path(pkg, pkg_roots, "")
+        if Path(path).exists() and not Path(path, "__init__.py").exists():
+            yield (pkg, [path])
+
+
 def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]:
     output = dict(pkg_roots.copy())
 
@@ -491,59 +505,81 @@ def _get_root(self):
 
 _FINDER_TEMPLATE = """\
 import sys
-from importlib.machinery import all_suffixes as module_suffixes
 from importlib.machinery import ModuleSpec
+from importlib.machinery import all_suffixes as module_suffixes
 from importlib.util import spec_from_file_location
 from itertools import chain
 from pathlib import Path
 
-class __EditableFinder:
-    MAPPING = {mapping!r}
-    NAMESPACES = {namespaces!r}
+MAPPING = {mapping!r}
+NAMESPACES = {namespaces!r}
+PATH_PLACEHOLDER = {name!r} + ".__path_hook__"
 
-    @classmethod
-    def find_spec(cls, fullname, path, target=None):
-        if fullname in cls.NAMESPACES:
-            return cls._namespace_spec(fullname)
 
-        for pkg, pkg_path in reversed(list(cls.MAPPING.items())):
+class _EditableFinder:  # MetaPathFinder
+    @classmethod
+    def find_spec(cls, fullname, path=None, target=None):
+        for pkg, pkg_path in reversed(list(MAPPING.items())):
             if fullname.startswith(pkg):
-                return cls._find_spec(fullname, pkg, pkg_path)
+                rest = fullname.replace(pkg, "").strip(".").split(".")
+                return cls._find_spec(fullname, Path(pkg_path, *rest))
 
         return None
 
     @classmethod
-    def _namespace_spec(cls, name):
-        # Since `cls` is appended to the path, this will only trigger
-        # when no other package is installed in the same namespace.
-        return ModuleSpec(name, None, is_package=True)
-        # ^-- PEP 451 mentions setting loader to None for namespaces.
-
-    @classmethod
-    def _find_spec(cls, fullname, parent, parent_path):
-        rest = fullname.replace(parent, "").strip(".").split(".")
-        candidate_path = Path(parent_path, *rest)
-
+    def _find_spec(cls, fullname, candidate_path):
         init = candidate_path / "__init__.py"
         candidates = (candidate_path.with_suffix(x) for x in module_suffixes())
         for candidate in chain([init], candidates):
             if candidate.exists():
-                spec = spec_from_file_location(fullname, candidate)
-                return spec
+                return spec_from_file_location(fullname, candidate)
 
-        if candidate_path.exists():
-            return cls._namespace_spec(fullname)
 
+class _EditableNamespaceFinder:  # PathEntryFinder
+    @classmethod
+    def _path_hook(cls, path):
+        if path == PATH_PLACEHOLDER:
+            return cls
+        raise ImportError
+
+    @classmethod
+    def _paths(cls, fullname):
+        # Ensure __path__ is not empty for the spec to be considered a namespace.
+        return NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER]
+
+    @classmethod
+    def find_spec(cls, fullname, target=None):
+        if fullname in NAMESPACES:
+            spec = ModuleSpec(fullname, None, is_package=True)
+            spec.submodule_search_locations = cls._paths(fullname)
+            return spec
+        return None
+
+    @classmethod
+    def find_module(cls, fullname):
         return None
 
 
 def install():
-    if not any(finder == __EditableFinder for finder in sys.meta_path):
-        sys.meta_path.append(__EditableFinder)
+    if not any(finder == _EditableFinder for finder in sys.meta_path):
+        sys.meta_path.append(_EditableFinder)
+
+    if not NAMESPACES:
+        return
+
+    if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks):
+        # PathEntryFinder is needed to create NamespaceSpec without private APIS
+        sys.path_hooks.append(_EditableNamespaceFinder._path_hook)
+    if PATH_PLACEHOLDER not in sys.path:
+        sys.path.append(PATH_PLACEHOLDER)  # Used just to trigger the path hook
 """
 
 
-def _finder_template(mapping: Mapping[str, str], namespaces: Set[str]):
-    """Create a string containing the code for a ``MetaPathFinder``."""
+def _finder_template(
+    name: str, mapping: Mapping[str, str], namespaces: Dict[str, List[str]]
+) -> str:
+    """Create a string containing the code for the``MetaPathFinder`` and
+    ``PathEntryFinder``.
+    """
     mapping = dict(sorted(mapping.items(), key=lambda p: p[0]))
-    return _FINDER_TEMPLATE.format(mapping=mapping, namespaces=namespaces)
+    return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces)

setuptools/tests/contexts.py

@@ -127,13 +127,13 @@ def session_locked_tmp_dir(request, tmp_path_factory, name):
 
 @contextlib.contextmanager
 def save_paths():
-    """Make sure initial ``sys.path`` and ``sys.meta_path`` are preserved"""
-    prev_paths = sys.path[:], sys.meta_path[:]
+    """Make sure ``sys.path``, ``sys.meta_path`` and ``sys.path_hooks`` are preserved"""
+    prev = sys.path[:], sys.meta_path[:], sys.path_hooks[:]
 
     try:
         yield
     finally:
-        sys.path, sys.meta_path = prev_paths
+        sys.path, sys.meta_path, sys.path_hooks = prev
 
 
 @contextlib.contextmanager

setuptools/tests/test_editable_install.py

@@ -7,6 +7,7 @@
 from importlib import import_module
 from pathlib import Path
 from textwrap import dedent
+from uuid import uuid4
 
 import jaraco.envs
 import jaraco.path
@@ -19,7 +20,8 @@
 from setuptools._importlib import resources as importlib_resources
 from setuptools.command.editable_wheel import (
     _LinkTree,
-    _find_mapped_namespaces,
+    _find_virtual_namespaces,
+    _find_namespaces,
     _find_package_roots,
     _finder_template,
 )
@@ -129,7 +131,7 @@ def test_editable_with_pyproject(tmp_path, venv, files, editable_mode):
     assert subprocess.check_output(cmd).strip() == b"3.14159.post0 foobar 42"
 
 
-def test_editable_with_flat_layout(tmp_path, venv, monkeypatch, editable_mode):
+def test_editable_with_flat_layout(tmp_path, venv, editable_mode):
     files = {
         "mypkg": {
             "pyproject.toml": dedent("""\
@@ -163,9 +165,7 @@ def test_editable_with_flat_layout(tmp_path, venv, monkeypatch, editable_mode):
 class TestLegacyNamespaces:
     """Ported from test_develop"""
 
-    def test_namespace_package_importable(
-        self, venv, tmp_path, monkeypatch, editable_mode
-    ):
+    def test_namespace_package_importable(self, venv, tmp_path, editable_mode):
         """
         Installing two packages sharing the same namespace, one installed
         naturally using pip or `--single-version-externally-managed`
@@ -184,9 +184,7 @@ def test_namespace_package_importable(
 
 
 class TestPep420Namespaces:
-    def test_namespace_package_importable(
-        self, venv, tmp_path, monkeypatch, editable_mode
-    ):
+    def test_namespace_package_importable(self, venv, tmp_path, editable_mode):
         """
         Installing two packages sharing the same namespace, one installed
         normally using pip and the other installed in editable mode
@@ -200,9 +198,7 @@ def test_namespace_package_importable(
         venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts])
         venv.run(["python", "-c", "import myns.n.pkgA; import myns.n.pkgB"])
 
-    def test_namespace_created_via_package_dir(
-        self, venv, tmp_path, monkeypatch, editable_mode
-    ):
+    def test_namespace_created_via_package_dir(self, venv, tmp_path, editable_mode):
         """Currently users can create a namespace by tweaking `package_dir`"""
         files = {
             "pkgA": {
@@ -305,7 +301,7 @@ def test_packages(self, tmp_path):
             "pkg1": str(tmp_path / "src1/pkg1"),
             "mod2": str(tmp_path / "src2/mod2")
         }
-        template = _finder_template(mapping, {})
+        template = _finder_template(str(uuid4()), mapping, {})
 
         with contexts.save_paths(), contexts.save_sys_modules():
             for mod in ("pkg1", "pkg1.subpkg", "pkg1.subpkg.mod1", "mod2"):
@@ -326,9 +322,9 @@ def test_namespace(self, tmp_path):
         jaraco.path.build(files, prefix=tmp_path)
 
         mapping = {"ns.othername": str(tmp_path / "pkg")}
-        namespaces = {"ns"}
+        namespaces = {"ns": []}
 
-        template = _finder_template(mapping, namespaces)
+        template = _finder_template(str(uuid4()), mapping, namespaces)
         with contexts.save_paths(), contexts.save_sys_modules():
             for mod in ("ns", "ns.othername"):
                 sys.modules.pop(mod, None)
@@ -344,7 +340,7 @@ def test_namespace(self, tmp_path):
             # Make sure resources can also be found
             assert text.read_text(encoding="utf-8") == "abc"
 
-    def test_combine_namespaces(self, tmp_path, monkeypatch):
+    def test_combine_namespaces(self, tmp_path):
         files = {
             "src1": {"ns": {"pkg1": {"__init__.py": "a = 13"}}},
             "src2": {"ns": {"mod2.py": "b = 37"}},
@@ -355,7 +351,8 @@ def test_combine_namespaces(self, tmp_path, monkeypatch):
             "ns.pkgA": str(tmp_path / "src1/ns/pkg1"),
             "ns": str(tmp_path / "src2/ns"),
         }
-        template = _finder_template(mapping, {})
+        namespaces_ = {"ns": [str(tmp_path / "src1"), str(tmp_path / "src2")]}
+        template = _finder_template(str(uuid4()), mapping, namespaces_)
 
         with contexts.save_paths(), contexts.save_sys_modules():
             for mod in ("ns", "ns.pkgA", "ns.mod2"):
@@ -370,6 +367,42 @@ def test_combine_namespaces(self, tmp_path, monkeypatch):
             assert pkgA.a == 13
             assert mod2.b == 37
 
+    def test_dynamic_path_computation(self, tmp_path):
+        # Follows the example in PEP 420
+        files = {
+            "project1": {"parent": {"child": {"one.py": "x = 1"}}},
+            "project2": {"parent": {"child": {"two.py": "x = 2"}}},
+            "project3": {"parent": {"child": {"three.py": "x = 3"}}},
+        }
+        jaraco.path.build(files, prefix=tmp_path)
+        mapping = {}
+        namespaces_ = {"parent": [str(tmp_path / "project1/parent")]}
+        template = _finder_template(str(uuid4()), mapping, namespaces_)
+
+        mods = (f"parent.child.{name}" for name in ("one", "two", "three"))
+        with contexts.save_paths(), contexts.save_sys_modules():
+            for mod in ("parent", "parent.child", "parent.child", *mods):
+                sys.modules.pop(mod, None)
+
+            self.install_finder(template)
+
+            one = import_module("parent.child.one")
+            assert one.x == 1
+
+            with pytest.raises(ImportError):
+                import_module("parent.child.two")
+
+            sys.path.append(str(tmp_path / "project2"))
+            two = import_module("parent.child.two")
+            assert two.x == 2
+
+            with pytest.raises(ImportError):
+                import_module("parent.child.three")
+
+            sys.path.append(str(tmp_path / "project3"))
+            three = import_module("parent.child.three")
+            assert three.x == 3
+
 
 def test_pkg_roots(tmp_path):
     """This test focus in getting a particular implementation detail right.
@@ -381,22 +414,43 @@ def test_pkg_roots(tmp_path):
         "d": {"__init__.py": "d = 1", "e": {"__init__.py": "de = 1"}},
         "f": {"g": {"h": {"__init__.py": "fgh = 1"}}},
         "other": {"__init__.py": "abc = 1"},
-        "another": {"__init__.py": "abcxy = 1"},
+        "another": {"__init__.py": "abcxyz = 1"},
+        "yet_another": {"__init__.py": "mnopq = 1"},
     }
     jaraco.path.build(files, prefix=tmp_path)
-    package_dir = {"a.b.c": "other", "a.b.c.x.y": "another"}
-    packages = ["a", "a.b", "a.b.c", "a.b.c.x.y", "d", "d.e", "f", "f.g", "f.g.h"]
+    package_dir = {
+        "a.b.c": "other",
+        "a.b.c.x.y.z": "another",
+        "m.n.o.p.q": "yet_another"
+    }
+    packages = [
+        "a",
+        "a.b",
+        "a.b.c",
+        "a.b.c.x.y",
+        "a.b.c.x.y.z",
+        "d",
+        "d.e",
+        "f",
+        "f.g",
+        "f.g.h",
+        "m.n.o.p.q",
+    ]
     roots = _find_package_roots(packages, package_dir, tmp_path)
     assert roots == {
         "a": str(tmp_path / "a"),
         "a.b.c": str(tmp_path / "other"),
-        "a.b.c.x.y": str(tmp_path / "another"),
+        "a.b.c.x.y.z": str(tmp_path / "another"),
         "d": str(tmp_path / "d"),
         "f": str(tmp_path / "f"),
+        "m.n.o.p.q": str(tmp_path / "yet_another"),
     }
 
-    namespaces = set(_find_mapped_namespaces(roots))
-    assert namespaces == {"a.b.c.x"}
+    ns = set(dict(_find_namespaces(packages, roots)))
+    assert ns == {"f", "f.g"}
+
+    ns = set(_find_virtual_namespaces(roots))
+    assert ns == {"a.b.c.x", "a.b.c.x.y", "m", "m.n", "m.n.o", "m.n.o.p"}
 
 
 class TestOverallBehaviour: