Support declarative requirements.

This adds support per the discussion we had on distutils-sig for
declarative requirements in setup.cfg.

Supported are requires-setup (the problem to be solved). requires-dist
(defined by d2to1, and needed as a consequences of supporting setup
requirements, because we can't run egg_info during the
pre-installation phase to detect requires. Similarly extras are
supported, for the same reason.

Neither setup-requires nor extras are defined by d2to1, so we may want
to bikeshed a little over the syntax there, but the basic structure
works.

docs/reference/pip_install.rst

@@ -337,9 +337,54 @@ the project path. This is one advantage over just using ``setup.py develop``,
 which creates the "egg-info" directly relative the current working directory.
 
 
+Build System Interface
+++++++++++++++++++++++
+
+In order for pip to install a package from source, pip must recognise the build
+system. Today only one build system is recognised, with two variants.
+
+If there is a ``setup.cfg`` with any of ``[extras]``, ``requires-dist`` or
+``requires-setup`` present then a declarative setuptools package is detected.
+
+Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed.
+
+Declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~
+
+``setup.py`` must implement the following commands::
+
+ setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
 
-Controlling setup_requires
-++++++++++++++++++++++++++
+With declarative dependencies, easy_install - the ``setup_requires`` keyword
+is never triggered, as pip can take care of installing the requirements before
+``setup.py`` is invoked.
+
+``setup.cfg`` must contain the package name, and one or more of setup requires,
+install requires or extra requires::
+
+ [metadata]
+ name = Example
+ requires-setup =
+ somedep
+ requires-dist =
+ runtimedep
+ [extras]
+ one =
+ anotherdep
+ tests =
+ mytestdep
+
+Non-declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``setup.py`` must implement the following commands::
+
+ setup.py egg_info [--egg-base XXX]
+ setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
+
+The ``egg_info`` command should create egg metadata for the package, as
+described in the setuptools documentation at
+http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags
 
 Setuptools offers the ``setup_requires`` `setup() keyword
 <http://pythonhosted.org/setuptools/setuptools.html#new-and-changed-setup-keywords>`_
@@ -370,19 +415,8 @@ To have the dependency located from a local directory and not crawl PyPI, add th
  allow_hosts = ''
  find_links = file:///path/to/local/archives
 
-
-Build System Interface
-++++++++++++++++++++++
-
-In order for pip to install a package from source, ``setup.py`` must implement
-the following commands::
-
- setup.py egg_info [--egg-base XXX]
- setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
-
-The ``egg_info`` command should create egg metadata for the package, as
-described in the setuptools documentation at
-http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags
+Common setuptools behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The ``install`` command should implement the complete process of installing the
 package to the target directory XXX.
@@ -404,13 +438,14 @@ Investigate in more detail when this command is required).
 
 No other build system commands are invoked by the ``pip install`` command.
 
+Wheels
+~~~~~~
+
 Installing a package from a wheel does not invoke the build system at all.
 
 .. _PyPI: http://pypi.python.org/pypi/
 .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
 
-
-
 .. _`pip install Options`:
 
 Options

pip/req/req_set.py

@@ -7,6 +7,7 @@
 
 from pip._vendor import pkg_resources
 from pip._vendor import requests
+from pip._vendor.six.moves import configparser
 
 from pip.download import (url_to_path, unpack_url)
 from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
@@ -80,6 +81,16 @@ def prep_for_dist(self):
  raise NotImplementedError(self.dist)
 
 
+def _sdist_or_static(req_to_install):
+ result = IsStaticMetadata(req_to_install)
+ try:
+ result.dist(None)
+ return result
+ except (configparser.NoSectionError, configparser.NoOptionError,
+ NoSetupCfgDeps):
+ return IsSDist(req_to_install)
+
+
 def make_abstract_dist(req_to_install):
  """Factory to make an abstract dist object.
 
@@ -89,11 +100,11 @@ def make_abstract_dist(req_to_install):
  :return: A concrete DistAbstraction.
  """
  if req_to_install.editable:
- return IsSDist(req_to_install)
+ return _sdist_or_static(req_to_install)
  elif req_to_install.link and req_to_install.link.is_wheel:
  return IsWheel(req_to_install)
  else:
- return IsSDist(req_to_install)
+ return _sdist_or_static(req_to_install)
 
 
 class IsWheel(DistAbstraction):
@@ -123,6 +134,85 @@ def prep_for_dist(self):
  self.req_to_install.assert_source_matches_version()
 
 
+class SetupCfgDistribution(pkg_resources.Distribution):
+ """A Distribution object that consults setup.cfg."""
+
+ def __init__(self, cfg, location=None):
+ self._cfg = cfg
+ project_name = cfg.get('metadata', 'name')
+ super(SetupCfgDistribution, self).__init__(
+ location=location, project_name=project_name)
+
+ @property
+ def _dep_map(self):
+ # Distribution._dep_map is not generic - its expressed in terms of the
+ # requires.txt format from within an egg info directory, and the
+ # metadata interfae within Distribution looks for files-and-lines.
+ # To provide metadata from a ConfigParser we could either marshall
+ # the data to a VFS, or we can reimplement this property which is
+ # the primary worker to obtain requirements.
+ try:
+ return self.__dep_map
+ except AttributeError:
+ # requires
+ try:
+ requires = self._cfg.get('metadata', 'requires-dist')
+ except configparser.NoOptionError:
+ requires = []
+ extras = (
+ self._cfg.has_section('extras')
+ and self._cfg.items('extras')) or []
+ dm = {}
+ dm.setdefault(None, []).extend(
+ pkg_resources.parse_requirements(requires))
+ for extra, extra_reqs in extras:
+ dm.setdefault(extra, []).extend(
+ pkg_resources.parse_requirements(extra_reqs))
+ self.__dep_map = dm
+ return dm
+
+ def setup_requires(self):
+ try:
+ setup_requires = self._cfg.get('metadata', 'requires-setup')
+ except configparser.NoOptionError:
+ return []
+ return pkg_resources.parse_requirements(setup_requires)
+
+
+class NoSetupCfgDeps(Exception):
+ pass
+
+
+class IsStaticMetadata(DistAbstraction):
+ """A static setup.cfg based source tree.
+
+ Must have a name and requires|setup_requires in setup.cfg to be usable.
+ We look for either requires or setup-requires to handle both the case where
+ a setup.py has setup-requires but no runtime requirements are declared, and
+ the case where folk have declaratively expressed their dist requirements
+ without needing a setup-requires.
+ """
+ def dist(self, finder):
+ cfg = configparser.SafeConfigParser()
+ setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg')
+ cfg.read(setup_cfg)
+ dist = SetupCfgDistribution(cfg=cfg, location=setup_cfg)
+ has_deps = False
+ for deps in dist._dep_map.values():
+ if deps:
+ has_deps = True
+ break
+ if not (dist.project_name or has_deps or dist.setup_requires()):
+ raise NoSetupCfgDeps()
+ return dist
+
+ def prep_for_dist(self):
+ self._dist = self.dist(None)
+ self.req_to_install.req = pkg_resources.Requirement.parse(
+ self._dist.project_name)
+ self.req_to_install._correct_build_location()
+
+
 class Installed(DistAbstraction):
 
  def dist(self, finder):
@@ -526,6 +616,15 @@ def add_req(subreq):
  self.add_requirement(req_to_install)
 
  if not self.ignore_dependencies:
+ if getattr(dist, 'setup_requires', None):
+ setup_requires = list(dist.setup_requires())
+ if setup_requires:
+ logger.debug(
+ "Installing setup_requires: %r",
+ ','.join(r.project_name for r in setup_requires))
+ for subreq in setup_requires:
+ add_req(subreq)
+
  if (req_to_install.extras):
  logger.debug(
  "Installing extra requirements: %r",

tests/data/packages/README.txt

@@ -71,6 +71,39 @@ priority-*
 ----------
 used for testing wheel priority over sdists
 
+SetupRequires
+-------------
+
+has a setup.cfg declaring a setup-requires on upper, and a setup.py that will
+fail to import if upper is not installed.
+
+SetupRequires-0.0.1.tar.gz
+--------------------------
+
+SetupRequires sdisted, for testing transitive setup_requires.
+
+SetupRequires2
+--------------
+
+has a setup.cfg declaring a dist-requires on setuprequires. Covers both
+setup-requires in depended-on packages, and setup.cfg with only requires-dist
+expressed.
+Also in setup.cfg declares two extras - a and b, a which brings in simple
+and b which brings in simple2, for testing extras from setup.cfg.
+
+SetupRequires2-0.0.1.tar.gz
+---------------------------
+
+SetupRequires2 sdisted, for testing declarative extras.
+
+SetupRequires3
+--------------
+
+requires SetupRequires2[a,b], as using extras for local paths is currently
+broken (issue 1236). Ideally SetupRequires3 would have the extras itself
+and no requires-dist (to test declarative extras as sole requirements).
+
+
 simple[2]-[123].0.tar.gz
 ------------------------
 contains "simple[2]" package; good for basic testing and version logic.

tests/data/packages/SetupRequires/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires
+requires-setup = 
+ upper

tests/data/packages/SetupRequires/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup
+import upper
+
+setup(
+ name='SetupRequires',
+ version='0.0.1',
+ packages=['setuprequires'],
+)

tests/data/packages/SetupRequires2/setup.cfg

@@ -0,0 +1,10 @@
+[metadata]
+name = SetupRequires2
+requires-dist = 
+ SetupRequires
+
+[extras]
+a = 
+ simple
+b =
+ simple2

tests/data/packages/SetupRequires2/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+ name='SetupRequires2',
+ version='0.0.1',
+ packages=['setuprequires2'],
+)

tests/data/packages/SetupRequires3/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires3
+requires-dist =
+ SetupRequires2[a,b]

tests/data/packages/SetupRequires3/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+ name='SetupRequires3',
+ version='0.0.1',
+ packages=['setuprequires3'],
+)

tests/functional/test_install.py

@@ -732,3 +732,35 @@ def test_install_upgrade_editable_depending_on_other_editable(script):
  script.pip('install', '--upgrade', '--editable', pkgb_path)
  result = script.pip('list')
  assert "pkgb" in result.stdout
+
+
+def _test_setup_requires(script, data, options, name):
+ to_install = data.packages.join(name)
+ args = ['install'] + options + [to_install, '-f', data.packages]
+ res = script.pip(*args, expect_error=False)
+ assert 'Running setup.py install for upper\n' in str(res)
+ return res
+
+
+def test_install_declarative_setup_requires_editable(script, data):
+ res = _test_setup_requires(script, data, ['-e'], 'SetupRequires')
+ assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_setup_requires(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires')
+ assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_requires(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires2')
+ assert script.site_packages / 'setuprequires2' in res.files_created, res
+
+
+def test_install_declarative_extras(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires3')
+ assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+ assert 'Running setup.py install for simple\n' in str(res), str(res)
+ assert 'Running setup.py install for simple2\n' in str(res), str(res)
+ assert 'Running setup.py install for SetupRequires3\n' in str(res), \
+ str(res)