Fix discovery of modules in namespace packages

docs/advanced.rst

@@ -51,11 +51,3 @@ Use the following configuration:
  # This requires `slotscheck` to be installed in that environment.
  #
  # language: system
-
-
-Namespace packages
-------------------
-
-Namespace packages come in `different flavors <https://packaging.python.org/en/latest/guides/packaging-namespace-packages/>`_.
-When using the ``-m/--module`` flag in the CLI, all these flavors are supported.
-When specifying file paths, *native* namespace packages are not supported.

docs/discovery.rst

@@ -9,7 +9,7 @@ However, there are some complications that you may need to be aware of.
 
  You should generally be fine if you follow these rules:
 
- - To check files in your current directory,
+ - To check files in your current directory, or subdirectories of it,
  you should run slotscheck as ``python -m slotscheck``.
  - To check files elsewhere, you may need to set the ``$PYTHONPATH``
  environment variable.

src/slotscheck/cli.py

@@ -42,6 +42,7 @@
 from .discovery import (
  AbsPath,
  FailedImport,
+ FileNotInSysPathError,
  ModuleLocated,
  ModuleName,
  ModuleTree,
@@ -162,9 +163,9 @@ def root(
 
  try:
  classes, modules = _collect(files, module, conf)
- except ModuleNotFoundError as e:
+ except (ModuleNotFoundError, FileNotInSysPathError) as e:
  print(
- f"ERROR: Module '{e.name}' not found.\n\n"
+ f"ERROR: {e}.\n\n"
  "See slotscheck.rtfd.io/en/latest/discovery.html\n"
  "for help resolving common import problems."
  )

src/slotscheck/discovery.py

@@ -2,11 +2,12 @@
 
 import importlib
 import pkgutil
+import sys
 from dataclasses import dataclass, field, replace
 from functools import partial, reduce
 from importlib.util import find_spec
 from inspect import isclass
-from itertools import chain, takewhile
+from itertools import chain
 from pathlib import Path
 from textwrap import indent
 from types import ModuleType
@@ -164,7 +165,7 @@ def module_tree(
  except BaseException as e:
  return FailedImport(module, e)
  if spec is None:
- raise ModuleNotFoundError(f"No module named '{module}'", name=module)
+ raise ModuleNotFoundError(f"No module named {module!r}", name=module)
  *namespaces, name = module.split(".")
  location = Path(spec.origin) if spec.has_location and spec.origin else None
  tree: ModuleTree
@@ -291,6 +292,12 @@ class ModuleLocated(NamedTuple):
  expected_location: Optional[AbsPath]
 
 
+class FileNotInSysPathError(Exception):
+ def __init__(self, file: Path) -> None:
+ super().__init__(f"File {str(file)!r} is not in PYTHONPATH")
+ self.file = file
+
+
 def _is_module(p: AbsPath) -> bool:
  return (p.is_file() and p.suffixes == [".py"]) or _is_package(p)
 
@@ -299,15 +306,32 @@ def _is_package(p: AbsPath) -> bool:
  return p.is_dir() and (p / _INIT_PY).is_file()
 
 
-def find_modules(p: AbsPath) -> Iterable[ModuleLocated]:
- "Recursively find modules at given path. Nonexistent Path is ignored"
+def _module_parents(
+ p: AbsPath, sys_path: FrozenSet[AbsPath]
+) -> Iterable[AbsPath]:
+ yield p
+ for pp in p.parents:
+ if pp in sys_path:
+ return
+ yield pp
+ raise FileNotInSysPathError(p)
+
+
+def _find_modules(
+ p: AbsPath, sys_path: FrozenSet[AbsPath]
+) -> Iterable[ModuleLocated]:
  if p.name == _INIT_PY:
- yield from find_modules(p.parent)
+ yield from _find_modules(p.parent, sys_path)
  elif _is_module(p):
- parents = [p] + list(takewhile(_is_package, p.parents))
+ parents = list(_module_parents(p, sys_path))
  yield ModuleLocated(
  ".".join(p.stem for p in reversed(parents)),
- (p / "__init__.py" if _is_package(p) else p),
+ (p / _INIT_PY if _is_package(p) else p),
  )
  elif p.is_dir():
- yield from flatten(map(find_modules, p.iterdir()))
+ yield from flatten(_find_modules(cp, sys_path) for cp in p.iterdir())
+
+
+def find_modules(p: AbsPath) -> Iterable[ModuleLocated]:
+ "Recursively find modules at given path. Nonexistent Path is ignored"
+ return _find_modules(p, frozenset(map(Path, sys.path)))

tests/src/conftest.py

@@ -13,9 +13,9 @@
 @pytest.fixture(scope="session", autouse=True)
 def add_pypath() -> Iterator[None]:
  "Add example modules to the python path"
- sys.path.insert(0, str(EXAMPLES_DIR))
+ sys.path[:0] = [str(EXAMPLES_DIR), str(EXAMPLES_DIR / "other")]
  yield
- sys.path.remove(str(EXAMPLES_DIR))
+ del sys.path[:2]
 
 
 @pytest.fixture(autouse=True)

tests/src/test_cli.py

@@ -1,4 +1,5 @@
 import os
+import re
 from importlib.util import find_spec
 from pathlib import Path
 
@@ -33,12 +34,26 @@ def test_module_doesnt_exist(runner: CliRunner):
  assert result.exit_code == 1
  assert isinstance(result.exception, SystemExit)
  assert result.output == (
- "ERROR: Module 'foo' not found.\n\n"
+ "ERROR: No module named 'foo'.\n\n"
  "See slotscheck.rtfd.io/en/latest/discovery.html\n"
  "for help resolving common import problems.\n"
  )
 
 
+def test_python_file_not_in_sys_path(runner: CliRunner, tmp_path: Path):
+ file = tmp_path / "foo.py"
+ file.write_text('print("Hello, world!")', encoding="utf-8")
+ result = runner.invoke(cli, [str(file)])
+ assert result.exit_code == 1
+ assert isinstance(result.exception, SystemExit)
+ assert re.fullmatch(
+ "ERROR: File '.*/foo.py' is not in PYTHONPATH.\n\n"
+ "See slotscheck.rtfd.io/en/latest/discovery.html\n"
+ "for help resolving common import problems.\n",
+ result.output,
+ )
+
+
 def test_module_is_uninspectable(runner: CliRunner):
  result = runner.invoke(cli, ["-m", "broken.submodule"])
  assert result.exit_code == 1
@@ -157,6 +172,16 @@ def test_multiple_modules(runner: CliRunner):
  assert result.output == "All OK!\nScanned 11 module(s), 70 class(es).\n"
 
 
+def test_implicitly_namespaced_path(runner: CliRunner):
+ result = runner.invoke(
+ cli,
+ [str(EXAMPLES_DIR / "implicitly_namespaced")],
+ catch_exceptions=False,
+ )
+ assert result.exit_code == 0
+ assert result.output == "All OK!\nScanned 7 module(s), 1 class(es).\n"
+
+
 def test_multiple_paths(runner: CliRunner):
  result = runner.invoke(
  cli,

tests/src/test_discovery.py

@@ -1,10 +1,12 @@
+from pathlib import Path
 from typing import FrozenSet, List, TypeVar
 from unittest import mock
 
 import pytest
 
 from slotscheck.discovery import (
  FailedImport,
+ FileNotInSysPathError,
  Module,
  ModuleLocated,
  ModuleTree,
@@ -370,32 +372,34 @@ def test_given_nonpython_file(self):
  def test_given_python_file(self):
  location = EXAMPLES_DIR / "files/subdir/myfile.py"
  result = list(find_modules(location))
- assert result == [ModuleLocated("myfile", location)]
+ assert result == [ModuleLocated("files.subdir.myfile", location)]
 
  def test_given_python_root_module(self):
  location = EXAMPLES_DIR / "files/subdir/some_module/"
  result = list(find_modules(location))
  assert result == [
- ModuleLocated("some_module", location / "__init__.py")
+ ModuleLocated("files.subdir.some_module", location / "__init__.py")
  ]
 
  def test_given_dir_containing_python_files(self):
  location = EXAMPLES_DIR / "files/my_scripts/"
  result = list(find_modules(location))
  assert len(result) == 4
  assert set(result) == {
- ModuleLocated("bla", location / "bla.py"),
- ModuleLocated("foo", location / "foo.py"),
- ModuleLocated("foo", location / "sub/foo.py"),
- ModuleLocated("mymodule", location / "mymodule/__init__.py"),
+ ModuleLocated("files.my_scripts.bla", location / "bla.py"),
+ ModuleLocated("files.my_scripts.foo", location / "foo.py"),
+ ModuleLocated("files.my_scripts.sub.foo", location / "sub/foo.py"),
+ ModuleLocated(
+ "files.my_scripts.mymodule", location / "mymodule/__init__.py"
+ ),
  }
 
  def test_given_file_within_module(self):
  location = EXAMPLES_DIR / "files/subdir/some_module/sub/foo.py"
  result = list(find_modules(location))
  assert result == [
  ModuleLocated(
- "some_module.sub.foo",
+ "files.subdir.some_module.sub.foo",
  EXAMPLES_DIR / "files/subdir/some_module/sub/foo.py",
  )
  ]
@@ -404,13 +408,23 @@ def test_given_submodule(self):
  location = EXAMPLES_DIR / "files/subdir/some_module/sub"
  result = list(find_modules(location))
  assert result == [
- ModuleLocated("some_module.sub", location / "__init__.py")
+ ModuleLocated(
+ "files.subdir.some_module.sub", location / "__init__.py"
+ )
  ]
 
  def test_given_init_py(self):
  location = EXAMPLES_DIR / "files/subdir/some_module/sub/__init__.py"
  result = list(find_modules(location))
- assert result == [ModuleLocated("some_module.sub", location)]
+ assert result == [
+ ModuleLocated("files.subdir.some_module.sub", location)
+ ]
+
+ def test_given_file_not_in_sys_path(self, tmp_path: Path):
+ location = tmp_path / "foo.py"
+ location.touch()
+ with pytest.raises(FileNotInSysPathError, match=r"foo\.py"):
+ list(find_modules(location))
 
 
 class TestConsolidate: