PEP 660 (build_editable) support

news/8212.feature.rst

@@ -0,0 +1,2 @@
+Support editable installs for projects that have a ``pyproject.toml`` and use a
+build backend that supports :pep:`660`.

src/pip/_internal/commands/install.py

@@ -306,6 +306,12 @@ def run(self, options: Values, args: List[str]) -> int:
  try:
  reqs = self.get_requirements(args, options, finder, session)
 
+ # Only when installing is it permitted to use PEP 660.
+ # In other circumstances (pip wheel, pip download) we generate
+ # regular (i.e. non editable) metadata and wheels.
+ for req in reqs:
+ req.permit_editable_wheels = True
+
  reject_location_related_install_options(reqs, options.install_options)
 
  preparer = self.make_requirement_preparer(
@@ -361,22 +367,22 @@ def run(self, options: Values, args: List[str]) -> int:
  global_options=[],
  )
 
- # If we're using PEP 517, we cannot do a direct install
+ # If we're using PEP 517, we cannot do a legacy setup.py install
  # so we fail here.
  pep517_build_failure_names: List[str] = [
  r.name for r in build_failures if r.use_pep517 # type: ignore
  ]
  if pep517_build_failure_names:
  raise InstallationError(
- "Could not build wheels for {} which use"
- " PEP 517 and cannot be installed directly".format(
+ "Could not build wheels for {}, which is required to "
+ "install pyproject.toml-based projects".format(
  ", ".join(pep517_build_failure_names)
  )
  )
 
  # For now, we just warn about failures building legacy
- # requirements, as we'll fall through to a direct
- # install for those.
+ # requirements, as we'll fall through to a setup.py install for
+ # those.
  for r in build_failures:
  if not r.use_pep517:
  r.legacy_install_reason = 8368

src/pip/_internal/distributions/sdist.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Set, Tuple
+from typing import Iterable, Set, Tuple
 
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
@@ -37,23 +37,17 @@ def prepare_distribution_metadata(
  self.req.prepare_metadata()
 
  def _setup_isolation(self, finder: PackageFinder) -> None:
- def _raise_conflicts(
- conflicting_with: str, conflicting_reqs: Set[Tuple[str, str]]
- ) -> None:
- format_string = (
- "Some build dependencies for {requirement} "
- "conflict with {conflicting_with}: {description}."
- )
- error_message = format_string.format(
- requirement=self.req,
- conflicting_with=conflicting_with,
- description=", ".join(
- f"{installed} is incompatible with {wanted}"
- for installed, wanted in sorted(conflicting)
- ),
- )
- raise InstallationError(error_message)
+ self._prepare_build_backend(finder)
+ # Install any extra build dependencies that the backend requests.
+ # This must be done in a second pass, as the pyproject.toml
+ # dependencies must be installed before we can call the backend.
+ if self.req.editable and self.req.permit_editable_wheels:
+ build_reqs = self._get_build_requires_editable()
+ else:
+ build_reqs = self._get_build_requires_wheel()
+ self._install_build_reqs(finder, build_reqs)
 
+ def _prepare_build_backend(self, finder: PackageFinder) -> None:
  # Isolate in a BuildEnvironment and install the build-time
  # requirements.
  pyproject_requires = self.req.pyproject_requires
@@ -67,7 +61,7 @@ def _raise_conflicts(
  self.req.requirements_to_check
  )
  if conflicting:
- _raise_conflicts("PEP 517/518 supported requirements", conflicting)
+ self._raise_conflicts("PEP 517/518 supported requirements", conflicting)
  if missing:
  logger.warning(
  "Missing build requirements in pyproject.toml for %s.",
@@ -78,19 +72,46 @@ def _raise_conflicts(
  "pip cannot fall back to setuptools without %s.",
  " and ".join(map(repr, sorted(missing))),
  )
- # Install any extra build dependencies that the backend requests.
- # This must be done in a second pass, as the pyproject.toml
- # dependencies must be installed before we can call the backend.
+
+ def _get_build_requires_wheel(self) -> Iterable[str]:
  with self.req.build_env:
  runner = runner_with_spinner_message("Getting requirements to build wheel")
  backend = self.req.pep517_backend
  assert backend is not None
  with backend.subprocess_runner(runner):
- reqs = backend.get_requires_for_build_wheel()
+ return backend.get_requires_for_build_wheel()
 
+ def _get_build_requires_editable(self) -> Iterable[str]:
+ with self.req.build_env:
+ runner = runner_with_spinner_message(
+ "Getting requirements to build editable"
+ )
+ backend = self.req.pep517_backend
+ assert backend is not None
+ with backend.subprocess_runner(runner):
+ return backend.get_requires_for_build_editable()
+
+ def _install_build_reqs(self, finder: PackageFinder, reqs: Iterable[str]) -> None:
  conflicting, missing = self.req.build_env.check_requirements(reqs)
  if conflicting:
- _raise_conflicts("the backend dependencies", conflicting)
+ self._raise_conflicts("the backend dependencies", conflicting)
  self.req.build_env.install_requirements(
  finder, missing, "normal", "Installing backend dependencies"
  )
+
+ def _raise_conflicts(
+ self, conflicting_with: str, conflicting_reqs: Set[Tuple[str, str]]
+ ) -> None:
+ format_string = (
+ "Some build dependencies for {requirement} "
+ "conflict with {conflicting_with}: {description}."
+ )
+ error_message = format_string.format(
+ requirement=self.req,
+ conflicting_with=conflicting_with,
+ description=", ".join(
+ f"{installed} is incompatible with {wanted}"
+ for installed, wanted in sorted(conflicting_reqs)
+ ),
+ )
+ raise InstallationError(error_message)

src/pip/_internal/operations/build/metadata.py

@@ -23,7 +23,9 @@ def generate_metadata(build_env: BuildEnvironment, backend: Pep517HookCaller) ->
  # Note that Pep517HookCaller implements a fallback for
  # prepare_metadata_for_build_wheel, so we don't have to
  # consider the possibility that this hook doesn't exist.
- runner = runner_with_spinner_message("Preparing wheel metadata")
+ runner = runner_with_spinner_message(
+ "Preparing wheel metadata (pyproject.toml)"
+ )
  with backend.subprocess_runner(runner):
  distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
 

src/pip/_internal/operations/build/metadata_editable.py

@@ -0,0 +1,34 @@
+"""Metadata generation logic for source distributions.
+"""
+
+import os
+
+from pip._vendor.pep517.wrappers import Pep517HookCaller
+
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+def generate_editable_metadata(
+ build_env: BuildEnvironment, backend: Pep517HookCaller
+) -> str:
+ """Generate metadata using mechanisms described in PEP 660.
+
+ Returns the generated metadata directory.
+ """
+ metadata_tmpdir = TempDirectory(kind="modern-metadata", globally_managed=True)
+
+ metadata_dir = metadata_tmpdir.path
+
+ with build_env:
+ # Note that Pep517HookCaller implements a fallback for
+ # prepare_metadata_for_build_wheel/editable, so we don't have to
+ # consider the possibility that this hook doesn't exist.
+ runner = runner_with_spinner_message(
+ "Preparing editable metadata (pyproject.toml)"
+ )
+ with backend.subprocess_runner(runner):
+ distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
+
+ return os.path.join(metadata_dir, distinfo_dir)

src/pip/_internal/operations/build/metadata_legacy.py

@@ -5,6 +5,7 @@
 import os
 
 from pip._internal.build_env import BuildEnvironment
+from pip._internal.cli.spinners import open_spinner
 from pip._internal.exceptions import InstallationError
 from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
 from pip._internal.utils.subprocess import call_subprocess
@@ -54,11 +55,13 @@ def generate_metadata(
  )
 
  with build_env:
- call_subprocess(
- args,
- cwd=source_dir,
- command_desc="python setup.py egg_info",
- )
+ with open_spinner("Preparing metadata (setup.py)") as spinner:
+ call_subprocess(
+ args,
+ cwd=source_dir,
+ command_desc="python setup.py egg_info",
+ spinner=spinner,
+ )
 
  # Return the .egg-info directory.
  return _find_egg_info(egg_info_dir)

src/pip/_internal/operations/build/wheel.py

@@ -23,7 +23,9 @@ def build_wheel_pep517(
  try:
  logger.debug("Destination directory: %s", tempd)
 
- runner = runner_with_spinner_message(f"Building wheel for {name} (PEP 517)")
+ runner = runner_with_spinner_message(
+ f"Building wheel for {name} (pyproject.toml)"
+ )
  with backend.subprocess_runner(runner):
  wheel_name = backend.build_wheel(
  tempd,

src/pip/_internal/operations/build/wheel_editable.py

@@ -0,0 +1,46 @@
+import logging
+import os
+from typing import Optional
+
+from pip._vendor.pep517.wrappers import HookMissing, Pep517HookCaller
+
+from pip._internal.utils.subprocess import runner_with_spinner_message
+
+logger = logging.getLogger(__name__)
+
+
+def build_wheel_editable(
+ name: str,
+ backend: Pep517HookCaller,
+ metadata_directory: str,
+ tempd: str,
+) -> Optional[str]:
+ """Build one InstallRequirement using the PEP 660 build process.
+
+ Returns path to wheel if successfully built. Otherwise, returns None.
+ """
+ assert metadata_directory is not None
+ try:
+ logger.debug("Destination directory: %s", tempd)
+
+ runner = runner_with_spinner_message(
+ f"Building editable for {name} (pyproject.toml)"
+ )
+ with backend.subprocess_runner(runner):
+ try:
+ wheel_name = backend.build_editable(
+ tempd,
+ metadata_directory=metadata_directory,
+ )
+ except HookMissing as e:
+ logger.error(
+ "Cannot build editable %s because the build "
+ "backend does not have the %s hook",
+ name,
+ e,
+ )
+ return None
+ except Exception:
+ logger.error("Failed building editable for %s", name)
+ return None
+ return os.path.join(tempd, wheel_name)

src/pip/_internal/req/constructors.py

@@ -22,7 +22,6 @@
 from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
-from pip._internal.pyproject import make_pyproject_path
 from pip._internal.req.req_file import ParsedRequirement
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.filetypes import is_archive_file
@@ -75,21 +74,6 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
  url_no_extras, extras = _strip_extras(url)
 
  if os.path.isdir(url_no_extras):
- setup_py = os.path.join(url_no_extras, "setup.py")
- setup_cfg = os.path.join(url_no_extras, "setup.cfg")
- if not os.path.exists(setup_py) and not os.path.exists(setup_cfg):
- msg = (
- 'File "setup.py" or "setup.cfg" not found. Directory cannot be '
- "installed in editable mode: {}".format(os.path.abspath(url_no_extras))
- )
- pyproject_path = make_pyproject_path(url_no_extras)
- if os.path.isfile(pyproject_path):
- msg += (
- '\n(A "pyproject.toml" file was found, but editable '
- "mode currently requires a setuptools-based build.)"
- )
- raise InstallationError(msg)
-
  # Treating it as code that has already been checked out
  url_no_extras = path_to_url(url_no_extras)
 
@@ -197,6 +181,7 @@ def install_req_from_editable(
  options: Optional[Dict[str, Any]] = None,
  constraint: bool = False,
  user_supplied: bool = False,
+ permit_editable_wheels: bool = False,
 ) -> InstallRequirement:
 
  parts = parse_req_from_editable(editable_req)
@@ -206,6 +191,7 @@ def install_req_from_editable(
  comes_from=comes_from,
  user_supplied=user_supplied,
  editable=True,
+ permit_editable_wheels=permit_editable_wheels,
  link=parts.link,
  constraint=constraint,
  use_pep517=use_pep517,