From 9c0d858f5a1e1de17eb9dcd04a7498a39c679051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 2 Dec 2022 14:40:01 -0800 Subject: [PATCH] Support for recursive extras MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- .pre-commit-config.yaml | 2 +- docs/changelog/2567.feature.rst | 1 + pyproject.toml | 1 + .../python/virtual_env/package/cmd_builder.py | 9 +++-- .../python/virtual_env/package/pyproject.py | 14 ++++++-- .../python/virtual_env/package/util.py | 33 +++++++++++++++---- .../package/test_python_package_util.py | 18 ++++++++-- tox.ini | 1 + 8 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 docs/changelog/2567.feature.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 78c005255a..937389ec9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: add-trailing-comma args: [--py36-plus] - repo: https://github.com/asottile/pyupgrade - rev: v3.2.2 + rev: v3.3.0 hooks: - id: pyupgrade args: ["--py37-plus"] diff --git a/docs/changelog/2567.feature.rst b/docs/changelog/2567.feature.rst new file mode 100644 index 0000000000..b3a9873029 --- /dev/null +++ b/docs/changelog/2567.feature.rst @@ -0,0 +1 @@ +Support for recursive extras in Python package dependencies - by :user:`gaborbernat`. diff --git a/pyproject.toml b/pyproject.toml index 0165c39985..fce580a45c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ optional-dependencies.testing = [ "covdefaults>=2.2", "devpi-client>=6.0.2", "devpi-server>=6.7", + "diff-cover>=7.2", "distlib>=0.3.6", "filelock>=3.8", "flaky>=3.7", diff --git a/src/tox/tox_env/python/virtual_env/package/cmd_builder.py b/src/tox/tox_env/python/virtual_env/package/cmd_builder.py index 6ec8d03590..720ecdeb6f 100644 --- a/src/tox/tox_env/python/virtual_env/package/cmd_builder.py +++ b/src/tox/tox_env/python/virtual_env/package/cmd_builder.py @@ -104,8 +104,10 @@ def perform_packaging(self, for_env: EnvConfigSet) -> list[Package]: def extract_install_info(self, for_env: EnvConfigSet, path: Path) -> list[Package]: extras: set[str] = for_env["extras"] if path.suffix == ".whl": - requires: list[str] = WheelDistribution(path).requires or [] - package: Package = WheelPackage(path, dependencies_with_extras([Requirement(i) for i in requires], extras)) + wheel_dist = WheelDistribution(path) + requires: list[str] = wheel_dist.requires or [] + deps = dependencies_with_extras([Requirement(i) for i in requires], extras, wheel_dist.name) + package: Package = WheelPackage(path, deps) else: # must be source distribution work_dir = self.env_tmp_dir / "sdist-extract" if work_dir.exists(): # pragma: no branch @@ -117,7 +119,8 @@ def extract_install_info(self, for_env: EnvConfigSet, path: Path) -> list[Packag with self._sdist_meta_tox_env.display_context(self._has_display_suspended): self._sdist_meta_tox_env.root = next(work_dir.iterdir()) # contains a single egg info folder deps = self._sdist_meta_tox_env.get_package_dependencies(for_env) - package = SdistPackage(path, dependencies_with_extras(deps, extras)) + name = self._sdist_meta_tox_env.get_package_name(for_env) + package = SdistPackage(path, dependencies_with_extras(deps, extras, name)) return [package] def register_run_env(self, run_env: RunToxEnv) -> Generator[tuple[str, str], PackageToxEnv, None]: diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index 11e412f6a1..dc3c7443cc 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -90,6 +90,7 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: self.builds: set[str] = set() self._distribution_meta: PathDistribution | None = None self._package_dependencies: list[Requirement] | None = None + self._package_name: str | None = None self._pkg_lock = RLock() # can build only one package at a time self.root = self.conf["package_root"] @@ -219,18 +220,20 @@ def _load_deps_from_built_metadata(self, for_env: EnvConfigSet) -> list[Requirem # to calculate the package metadata, otherwise ourselves of_type: str = for_env["package"] reqs: list[Requirement] | None = None + name = "" if of_type in ("wheel", "editable"): # wheel packages w_env = self._wheel_build_envs.get(for_env["wheel_build_env"]) if w_env is not None and w_env is not self: with w_env.display_context(self._has_display_suspended): if isinstance(w_env, Pep517VirtualEnvPackager): - reqs = w_env.get_package_dependencies(for_env) + reqs, name = w_env.get_package_dependencies(for_env), w_env.get_package_name(for_env) else: reqs = [] if reqs is None: reqs = self.get_package_dependencies(for_env) + name = self.get_package_name(for_env) extras: set[str] = for_env["extras"] - deps = dependencies_with_extras(reqs, extras) + deps = dependencies_with_extras(reqs, extras, name) return deps def get_package_dependencies(self, for_env: EnvConfigSet) -> list[Requirement]: @@ -241,6 +244,13 @@ def get_package_dependencies(self, for_env: EnvConfigSet) -> list[Requirement]: self._package_dependencies = [Requirement(i) for i in requires] # pragma: no branch return self._package_dependencies + def get_package_name(self, for_env: EnvConfigSet) -> str: + with self._pkg_lock: + if self._package_name is None: # pragma: no branch + self._ensure_meta_present(for_env) + self._package_name = cast(PathDistribution, self._distribution_meta).name + return self._package_name + def _ensure_meta_present(self, for_env: EnvConfigSet) -> None: if self._distribution_meta is not None: # pragma: no branch return # pragma: no cover diff --git a/src/tox/tox_env/python/virtual_env/package/util.py b/src/tox/tox_env/python/virtual_env/package/util.py index f9f485884c..b84f687a55 100644 --- a/src/tox/tox_env/python/virtual_env/package/util.py +++ b/src/tox/tox_env/python/virtual_env/package/util.py @@ -6,21 +6,44 @@ from packaging.requirements import Requirement -def dependencies_with_extras(deps: list[Requirement], extras: set[str]) -> list[Requirement]: +def dependencies_with_extras(deps: list[Requirement], extras: set[str], package_name: str) -> list[Requirement]: + deps = _normalize_req(deps) + result: list[Requirement] = [] + found: set[str] = set() + todo: set[str | None] = extras | {None} + visited: set[str | None] = set() + while todo: + new_extras: set[str | None] = set() + for req in deps: + if todo & (req.extras or {None}): # type: ignore[arg-type] + if req.name == package_name: # support for recursive extras + new_extras.update(req.extras or set()) + else: + req = deepcopy(req) + req.extras.clear() # strip the extra part as the installation will invoke it without + req_str = str(req) + if req_str not in found: + found.add(req_str) + result.append(req) + visited.update(todo) + todo = new_extras - visited + return result + + +def _normalize_req(deps: list[Requirement]) -> list[Requirement]: + # extras might show up as markers, move them into extras property result: list[Requirement] = [] for req in deps: req = deepcopy(req) markers: list[str | tuple[Variable, Variable, Variable]] = getattr(req.marker, "_markers", []) or [] - # find the extra marker (if has) _at: int | None = None - extra: str | None = None for _at, (marker_key, op, marker_value) in ( (_at_marker, marker) for _at_marker, marker in enumerate(markers) if isinstance(marker, tuple) and len(marker) == 3 ): if marker_key.value == "extra" and op.value == "==": # pragma: no branch - extra = marker_value.value + req.extras.add(marker_value.value) del markers[_at] _at -= 1 if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")): @@ -28,7 +51,5 @@ def dependencies_with_extras(deps: list[Requirement], extras: set[str]) -> list[ if len(markers) == 0: req.marker = None break - if not (extra is None or extra in extras): - continue result.append(req) return result diff --git a/tests/tox_env/python/virtual_env/package/test_python_package_util.py b/tests/tox_env/python/virtual_env/package/test_python_package_util.py index b3369a97c2..4ff9e348a6 100644 --- a/tests/tox_env/python/virtual_env/package/test_python_package_util.py +++ b/tests/tox_env/python/virtual_env/package/test_python_package_util.py @@ -25,7 +25,7 @@ def pkg_with_extras(pkg_with_extras_project: Path) -> PathDistribution: def test_load_dependency_no_extra(pkg_with_extras: PathDistribution) -> None: - result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], set()) + result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], set(), "") for left, right in zip_longest(result, (Requirement("platformdirs>=2.1"), Requirement("colorama>=0.4.3"))): assert isinstance(right, Requirement) assert str(left) == str(right) @@ -33,7 +33,7 @@ def test_load_dependency_no_extra(pkg_with_extras: PathDistribution) -> None: def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None: py_ver = ".".join(str(i) for i in sys.version_info[0:2]) - result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"}) + result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"}, "") exp = [ Requirement("platformdirs>=2.1"), Requirement("colorama>=0.4.3"), @@ -45,3 +45,17 @@ def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None: for left, right in zip_longest(result, exp): assert isinstance(right, Requirement) assert str(left) == str(right) + + +def test_loads_deps_recursive_extras() -> None: + requires = [ + Requirement("no-extra"), + Requirement("dep1[dev]"), + Requirement("dep1[test]"), + Requirement("dep2[test]"), + Requirement("dep3[docs]"), + Requirement("name[dev]"), + Requirement("name[test,dev]"), + ] + result = dependencies_with_extras(requires, {"dev"}, "name") + assert [str(i) for i in result] == ["no-extra", "dep1", "dep2"] diff --git a/tox.ini b/tox.ini index db96b509ec..099dca9469 100644 --- a/tox.ini +++ b/tox.ini @@ -32,6 +32,7 @@ commands = --cov-report xml:{toxworkdir}{/}coverage.{envname}.xml \ -n={env:PYTEST_XDIST_PROC_NR:auto} \ tests --durations 5 --run-integration} + diff-cover --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}{/}coverage.{envname}.xml package = wheel wheel_build_env = .pkg