Skip to content

Commit

Permalink
Merge pull request #1964 from hauntsaninja/static-pyproj
Browse files Browse the repository at this point in the history
Parse pyproject.toml statically where possible
  • Loading branch information
chrysle authored Mar 25, 2024
2 parents 6f2c9cd + 2098b99 commit b692edb
Show file tree
Hide file tree
Showing 5 changed files with 268 additions and 7 deletions.
94 changes: 93 additions & 1 deletion piptools/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@
import pyproject_hooks
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line, parse_req_from_line
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import Requirement

if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib

PYPROJECT_TOML = "pyproject.toml"

Expand All @@ -32,23 +39,95 @@ def get_all(self, name: str, failobj: None = None) -> list[Any] | None: ...
def get_all(self, name: str, failobj: _T) -> list[Any] | _T: ...


@dataclass
class StaticProjectMetadata:
extras: tuple[str, ...]
requirements: tuple[InstallRequirement, ...]


@dataclass
class ProjectMetadata:
extras: tuple[str, ...]
requirements: tuple[InstallRequirement, ...]
build_requirements: tuple[InstallRequirement, ...]


def maybe_statically_parse_project_metadata(
src_file: pathlib.Path,
) -> StaticProjectMetadata | None:
"""
Return the metadata for a project, if it can be statically parsed from ``pyproject.toml``.
This function is typically significantly faster than invoking a build backend.
Returns None if the project metadata cannot be statically parsed.
"""
if src_file.name != PYPROJECT_TOML:
return None

try:
with open(src_file, "rb") as f:
pyproject_contents = tomllib.load(f)
except tomllib.TOMLDecodeError:
return None

# Not valid PEP 621 metadata
if (
"project" not in pyproject_contents
or "name" not in pyproject_contents["project"]
):
return None

project_table = pyproject_contents["project"]

# Dynamic dependencies require build backend invocation
dynamic = project_table.get("dynamic", [])
if "dependencies" in dynamic or "optional-dependencies" in dynamic:
return None

package_name = project_table["name"]
comes_from = f"{package_name} ({src_file})"

extras = project_table.get("optional-dependencies", {}).keys()
install_requirements = [
InstallRequirement(Requirement(req), comes_from)
for req in project_table.get("dependencies", [])
]
for extra, reqs in (
pyproject_contents.get("project", {}).get("optional-dependencies", {}).items()
):
for req in reqs:
requirement = Requirement(req)
if requirement.name == package_name:
# Similar to logic for handling self-referential requirements
# from _prepare_requirements
requirement.url = src_file.parent.as_uri()
# Note we don't need to modify `requirement` to include this extra
marker = Marker(f"extra == '{extra}'")
install_requirements.append(
InstallRequirement(requirement, comes_from, markers=marker)
)

return StaticProjectMetadata(
extras=tuple(extras),
requirements=tuple(install_requirements),
)


def build_project_metadata(
src_file: pathlib.Path,
build_targets: tuple[str, ...],
*,
attempt_static_parse: bool,
isolated: bool,
quiet: bool,
) -> ProjectMetadata:
) -> ProjectMetadata | StaticProjectMetadata:
"""
Return the metadata for a project.
First, optionally attempt to determine the metadata statically from the
``pyproject.toml`` file. This will not work if build_targets are specified,
since we cannot determine build requirements statically.
Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata
if available, otherwise ``build_wheel``.
Expand All @@ -58,12 +137,25 @@ def build_project_metadata(
:param src_file: Project source file
:param build_targets: A tuple of build targets to get the dependencies
of (``sdist`` or ``wheel`` or ``editable``).
:param attempt_static_parse: Whether to attempt to statically parse the
project metadata from ``pyproject.toml``.
Cannot be used with ``build_targets``.
:param isolated: Whether to run invoke the backend in the current
environment or to create an isolated one and invoke it
there.
:param quiet: Whether to suppress the output of subprocesses.
"""

if attempt_static_parse:
if build_targets:
raise ValueError(
"Cannot execute the PEP 517 optional get_requires_for_build* "
"hooks statically, as build requirements are requested"
)
project_metadata = maybe_statically_parse_project_metadata(src_file)
if project_metadata is not None:
return project_metadata

src_dir = src_file.parent
with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder:
metadata = _build_project_wheel_metadata(builder)
Expand Down
4 changes: 3 additions & 1 deletion piptools/scripts/compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from pip._internal.utils.misc import redact_auth_from_url

from .._compat import parse_requirements
from ..build import build_project_metadata
from ..build import ProjectMetadata, build_project_metadata
from ..cache import DependencyCache
from ..exceptions import NoCandidateFound, PipToolsError
from ..logging import log
Expand Down Expand Up @@ -365,6 +365,7 @@ def cli(
metadata = build_project_metadata(
src_file=Path(src_file),
build_targets=build_deps_targets,
attempt_static_parse=not bool(build_deps_targets),
isolated=build_isolation,
quiet=log.verbosity <= 0,
)
Expand All @@ -378,6 +379,7 @@ def cli(
if all_extras:
extras += metadata.extras
if build_deps_targets:
assert isinstance(metadata, ProjectMetadata)
constraints.extend(metadata.build_requirements)
else:
constraints.extend(
Expand Down
105 changes: 103 additions & 2 deletions tests/test_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@

import pytest

from piptools.build import build_project_metadata
from piptools.build import (
ProjectMetadata,
StaticProjectMetadata,
build_project_metadata,
maybe_statically_parse_project_metadata,
)
from tests.constants import PACKAGES_PATH


Expand All @@ -25,8 +30,9 @@ def test_build_project_metadata_resolved_correct_build_dependencies(
shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
src_file = tmp_path / "setup.py"
metadata = build_project_metadata(
src_file, ("editable",), isolated=True, quiet=False
src_file, ("editable",), attempt_static_parse=False, isolated=True, quiet=False
)
assert isinstance(metadata, ProjectMetadata)
build_requirements = sorted(r.name for r in metadata.build_requirements)
assert build_requirements == [
"fake_dynamic_build_dep_for_all",
Expand All @@ -35,3 +41,98 @@ def test_build_project_metadata_resolved_correct_build_dependencies(
"setuptools",
"wheel",
]


def test_build_project_metadata_static(tmp_path):
"""Test static parsing branch of build_project_metadata"""
src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_pyproject"
shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
src_file = tmp_path / "pyproject.toml"
metadata = build_project_metadata(
src_file, (), attempt_static_parse=True, isolated=True, quiet=False
)
assert isinstance(metadata, StaticProjectMetadata)
requirements = [(r.name, r.extras, str(r.markers)) for r in metadata.requirements]
requirements.sort(key=lambda x: x[0])
assert requirements == [
("fake_direct_extra_runtime_dep", {"with_its_own_extra"}, 'extra == "x"'),
("fake_direct_runtime_dep", set(), "None"),
]
assert metadata.extras == ("x",)


def test_build_project_metadata_raises_error(tmp_path):
src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_build_deps"
shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
src_file = tmp_path / "setup.py"
with pytest.raises(
ValueError, match="Cannot execute the PEP 517 optional.* hooks statically"
):
build_project_metadata(
src_file,
("editable",),
attempt_static_parse=True,
isolated=True,
quiet=False,
)


def test_static_parse_valid(tmp_path):
src_file = tmp_path / "pyproject.toml"

valid = """
[project]
name = "foo"
version = "0.1.0"
dependencies = ["bar>=1"]
[project.optional-dependencies]
baz = ["qux[extra]"]
"""
src_file.write_text(valid)
metadata = maybe_statically_parse_project_metadata(src_file)
assert isinstance(metadata, StaticProjectMetadata)
assert [str(r.req) for r in metadata.requirements] == ["bar>=1", "qux[extra]"]
assert metadata.extras == ("baz",)


def test_static_parse_invalid(tmp_path):
src_file = tmp_path / "pyproject.toml"

invalid_toml = """this is not valid toml"""
src_file.write_text(invalid_toml)
assert maybe_statically_parse_project_metadata(src_file) is None

no_pep621 = """
[build-system]
requires = ["setuptools"]
"""
src_file.write_text(no_pep621)
assert maybe_statically_parse_project_metadata(src_file) is None

invalid_pep621 = """
[project]
# no name
version = "0.1.0"
"""
src_file.write_text(invalid_pep621)
assert maybe_statically_parse_project_metadata(src_file) is None

dynamic_deps = """
[project]
name = "foo"
dynamic = ["dependencies"]
"""
src_file.write_text(dynamic_deps)
assert maybe_statically_parse_project_metadata(src_file) is None

dynamic_optional_deps = """
[project]
name = "foo"
dynamic = ["optional-dependencies"]
"""
src_file.write_text(dynamic_optional_deps)
assert maybe_statically_parse_project_metadata(src_file) is None

src_file = tmp_path / "setup.py"
src_file.write_text("print('hello')")
assert maybe_statically_parse_project_metadata(src_file) is None
64 changes: 61 additions & 3 deletions tests/test_cli_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -3330,7 +3330,7 @@ def test_pass_pip_cache_to_pip_args(tmpdir, runner, current_resolver):


@backtracking_resolver_only
def test_compile_recursive_extras(runner, tmp_path, current_resolver):
def test_compile_recursive_extras_static(runner, tmp_path, current_resolver):
(tmp_path / "pyproject.toml").write_text(
dedent(
"""
Expand Down Expand Up @@ -3364,8 +3364,66 @@ def test_compile_recursive_extras(runner, tmp_path, current_resolver):
small-fake-a==0.2
small-fake-b==0.3
"""
assert out.exit_code == 0
assert expected == out.stdout
try:
assert out.exit_code == 0
assert expected == out.stdout
except Exception: # pragma: no cover
print(out.stdout)
print(out.stderr)
raise


@backtracking_resolver_only
def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolver):
(tmp_path / "pyproject.toml").write_text(
dedent(
"""
[project]
name = "foo"
version = "0.0.1"
dependencies = ["small-fake-a"]
[project.optional-dependencies]
footest = ["small-fake-b"]
dev = ["foo[footest]"]
"""
)
)
(tmp_path / "constraints.txt").write_text("wheel<0.43")
out = runner.invoke(
cli,
[
"--no-build-isolation",
"--no-header",
"--no-annotate",
"--no-emit-options",
"--extra",
"dev",
"--build-deps-for",
"wheel",
"--find-links",
os.fspath(MINIMAL_WHEELS_PATH),
os.fspath(tmp_path / "pyproject.toml"),
"--constraint",
os.fspath(tmp_path / "constraints.txt"),
"--output-file",
"-",
],
)
expected = rf"""foo[footest] @ {tmp_path.as_uri()}
small-fake-a==0.2
small-fake-b==0.3
wheel==0.42.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools
"""
try:
assert out.exit_code == 0
assert expected == out.stdout
except Exception: # pragma: no cover
print(out.stdout)
print(out.stderr)
raise


def test_config_option(pip_conf, runner, tmp_path, make_config_file):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[project]
name="small_fake_with_pyproject"
version=0.1
dependencies=[
"fake_direct_runtime_dep",
]
[project.optional-dependencies]
x = ["fake_direct_extra_runtime_dep[with_its_own_extra]"]

0 comments on commit b692edb

Please sign in to comment.