Support declarative requirements.

This adds support per the discussion we had on distutils-sig for
declarative requirements in setup.cfg.

Supported are requires-setup (the problem to be solved). requires-dist
(defined by d2to1, and needed as a consequences of supporting setup
requirements, because we can't run egg_info during the
pre-installation phase to detect requires. Similarly extras are
supported, for the same reason.

Neither setup-requires nor extras are defined by d2to1, so we may want
to bikeshed a little over the syntax there, but the basic structure
works.

docs/reference/pip_install.rst

@@ -337,45 +337,28 @@ the project path. This is one advantage over just using ``setup.py develop``,
 which creates the "egg-info" directly relative the current working directory.
 
 
+Build System Interface
+++++++++++++++++++++++
 
-Controlling setup_requires
-++++++++++++++++++++++++++
-
-Setuptools offers the ``setup_requires`` `setup() keyword
-<http://pythonhosted.org/setuptools/setuptools.html#new-and-changed-setup-keywords>`_
-for specifying dependencies that need to be present in order for the `setup.py`
-script to run. Internally, Setuptools uses ``easy_install`` to fulfill these
-dependencies.
-
-pip has no way to control how these dependencies are located. None of the
-Package Index Options have an effect.
-
-The solution is to configure a "system" or "personal" `Distutils configuration
-file
-<http://docs.python.org/2/install/index.html#distutils-configuration-files>`_ to
-manage the fulfillment.
-
-For example, to have the dependency located at an alternate index, add this:
-
-::
+In order for pip to install a package from source, pip must recognise the build
+system. Today only one build system is recognised, with two variants.
 
- [easy_install]
- index_url = https://my.index-mirror.com
+If there is a ``setup.cfg`` with any of ``[extras]``, ``requires-dist`` or
+``requires-setup`` present then a declarative setuptools package is detected.
 
-To have the dependency located from a local directory and not crawl PyPI, add this:
+Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed.
 
-::
+Declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~
 
- [easy_install]
- allow_hosts = ''
- find_links = file:///path/to/local/archives
+``setup.py`` must implement the following commands::
 
+ setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
 
-Build System Interface
-++++++++++++++++++++++
+Non-declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-In order for pip to install a package from source, ``setup.py`` must implement
-the following commands::
+``setup.py`` must implement the following commands::
 
  setup.py egg_info [--egg-base XXX]
  setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
@@ -409,6 +392,39 @@ Installing a package from a wheel does not invoke the build system at all.
 .. _PyPI: http://pypi.python.org/pypi/
 .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
 
+Controlling setup_requires()
+++++++++++++++++++++++++++++
+
+Setuptools offers the ``setup_requires`` `setup() keyword
+<http://pythonhosted.org/setuptools/setuptools.html#new-and-changed-setup-keywords>`_
+for specifying dependencies that need to be present in order for the `setup.py`
+script to run. Internally, Setuptools uses ``easy_install`` to fulfill these
+dependencies.
+
+pip has no way to control how these dependencies are located. None of the
+Package Index Options have an effect.
+
+The solution is to configure a "system" or "personal" `Distutils configuration
+file
+<http://docs.python.org/2/install/index.html#distutils-configuration-files>`_ to
+manage the fulfillment.
+
+For example, to have the dependency located at an alternate index, add this:
+
+::
+
+ [easy_install]
+ index_url = https://my.index-mirror.com
+
+To have the dependency located from a local directory and not crawl PyPI, add this:
+
+::
+
+ [easy_install]
+ allow_hosts = ''
+ find_links = file:///path/to/local/archives
+
+
 
 
 .. _`pip install Options`:

pip/req/req_set.py

@@ -7,6 +7,7 @@
 
 from pip._vendor import pkg_resources
 from pip._vendor import requests
+from pip._vendor.six.moves import configparser
 
 from pip.download import (url_to_path, unpack_url)
 from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
@@ -80,6 +81,15 @@ def prep_for_dist(self):
  raise NotImplementedError(self.dist)
 
 
+def _sdist_or_static(req_to_install):
+ result = IsStaticMetadata(req_to_install)
+ try:
+ result.dist(None)
+ return result
+ except (configparser.NoSectionError, configparser.NoOptionError, AssertionError):
+ return IsSDist(req_to_install)
+
+
 def make_abstract_dist(req_to_install):
  """Factory to make an abstract dist object.
 
@@ -89,11 +99,11 @@ def make_abstract_dist(req_to_install):
  :return: A concrete DistAbstraction.
  """
  if req_to_install.editable:
- return IsSDist(req_to_install)
+ return _sdist_or_static(req_to_install)
  elif req_to_install.link and req_to_install.link.is_wheel:
  return IsWheel(req_to_install)
  else:
- return IsSDist(req_to_install)
+ return _sdist_or_static(req_to_install)
 
 
 class IsWheel(DistAbstraction):
@@ -123,6 +133,68 @@ def prep_for_dist(self):
  self.req_to_install.assert_source_matches_version()
 
 
+class SetupCfgDistribution(pkg_resources.Distribution):
+
+ def __init__(self, cfg, location=None):
+ self._cfg = cfg
+ project_name = cfg.get('metadata', 'name')
+ super(SetupCfgDistribution, self).__init__(
+ location=location, project_name=project_name)
+
+ @property
+ def _dep_map(self):
+ try:
+ return self.__dep_map
+ except AttributeError:
+ # requires
+ try:
+ requires = self._cfg.get('metadata', 'requires-dist')
+ except configparser.NoOptionError:
+ requires = []
+ extras = (
+ self._cfg.has_section('extras')
+ and self._cfg.items('extras')) or []
+ dm = {}
+ dm.setdefault(None, []).extend(
+ pkg_resources.parse_requirements(requires))
+ for extra, extra_reqs in extras:
+ dm.setdefault(extra, []).extend(
+ pkg_resources.parse_requirements(extra_reqs))
+ return dm
+
+ def setup_requires(self):
+ try:
+ setup_requires = self._cfg.get('metadata', 'requires-setup')
+ except configparser.NoOptionError:
+ return []
+ return pkg_resources.parse_requirements(setup_requires)
+
+
+class IsStaticMetadata(DistAbstraction):
+ """A static setup.cfg based source tree.
+
+ Must have a name and requires|setup_requires in setup.cfg to be usable.
+ We look for either requires or setup-requires to handle both the case where
+ a setup.py has setup-requires but no runtime requirements are declared, and
+ the case where folk have declaratively expressed their dist requirements
+ without needing a setup-requires.
+ """
+ def dist(self, finder):
+ cfg = configparser.SafeConfigParser()
+ setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg')
+ cfg.read(setup_cfg)
+ dist = SetupCfgDistribution(cfg = cfg, location=setup_cfg)
+ assert dist.project_name
+ assert dist.requires() or dist.setup_requires()
+ return dist
+
+ def prep_for_dist(self):
+ self._dist = self.dist(None)
+ self.req_to_install.req = pkg_resources.Requirement.parse(
+ self._dist.project_name)
+ self.req_to_install._correct_build_location()
+
+
 class Installed(DistAbstraction):
 
  def dist(self, finder):
@@ -518,6 +590,15 @@ def add_req(subreq):
  self.add_requirement(req_to_install)
 
  if not self.ignore_dependencies:
+ if getattr(dist, 'setup_requires', None):
+ setup_requires = list(dist.setup_requires())
+ if setup_requires:
+ logger.debug(
+ "Installing setup_requires: %r",
+ ','.join(r.project_name for r in setup_requires))
+ for subreq in setup_requires:
+ add_req(subreq)
+
  if (req_to_install.extras):
  logger.debug(
  "Installing extra requirements: %r",

tests/data/packages/README.txt

@@ -71,6 +71,39 @@ priority-*
 ----------
 used for testing wheel priority over sdists
 
+SetupRequires
+-------------
+
+has a setup.cfg declaring a setup-requires on upper, and a setup.py that will
+fail to import if upper is not installed.
+
+SetupRequires-0.0.1.tar.gz
+--------------------------
+
+SetupRequires sdisted, for testing transitive setup_requires.
+
+SetupRequires2
+--------------
+
+has a setup.cfg declaring a dist-requires on setuprequires. Covers both
+setup-requires in depended-on packages, and setup.cfg with only requires-dist
+expressed.
+Also in setup.cfg declares two extras - a and b, a which brings in simple
+and b which brings in simple2, for testing extras from setup.cfg.
+
+SetupRequires2-0.0.1.tar.gz
+---------------------------
+
+SetupRequires2 sdisted, for testing declarative extras.
+
+SetupRequires3
+--------------
+
+requires SetupRequires2[a,b], as using extras for local paths is currently
+broken (issue 1236). Ideally SetupRequires3 would have the extras itself
+and no requires-dist (to test declarative extras as sole requirements).
+
+
 simple[2]-[123].0.tar.gz
 ------------------------
 contains "simple[2]" package; good for basic testing and version logic.

tests/data/packages/SetupRequires/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires
+requires-setup = 
+ upper

tests/data/packages/SetupRequires/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup
+import upper
+
+setup(
+ name='SetupRequires',
+ version='0.0.1',
+ packages=['setuprequires'],
+)

tests/data/packages/SetupRequires2/setup.cfg

@@ -0,0 +1,10 @@
+[metadata]
+name = SetupRequires2
+requires-dist = 
+ SetupRequires
+
+[extras]
+a = 
+ simple
+b =
+ simple2

tests/data/packages/SetupRequires2/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+ name='SetupRequires2',
+ version='0.0.1',
+ packages=['setuprequires2'],
+)

tests/data/packages/SetupRequires3/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires3
+requires-dist =
+ SetupRequires2[a,b]

tests/data/packages/SetupRequires3/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+ name='SetupRequires3',
+ version='0.0.1',
+ packages=['setuprequires3'],
+)

tests/functional/test_install.py

@@ -769,3 +769,34 @@ def test_install_upgrade_editable_depending_on_other_editable(script):
  script.pip('install', '--upgrade', '--editable', pkgb_path)
  result = script.pip('list')
  assert "pkgb" in result.stdout
+
+
+def _test_setup_requires(script, data, options, name):
+ to_install = data.packages.join(name)
+ args = ['install'] + options + [to_install, '-f', data.packages]
+ res = script.pip(*args, expect_error=False)
+ assert 'Running setup.py install for upper\n' in str(res)
+ return res
+
+
+def test_install_declarative_setup_requires_editable(script, data):
+ res = _test_setup_requires(script, data, ['-e'], 'SetupRequires')
+ assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_setup_requires(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires')
+ assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_requires(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires2')
+ assert script.site_packages / 'setuprequires2' in res.files_created, res
+
+
+def test_install_declarative_extras(script, data):
+ res = _test_setup_requires(script, data, [], 'SetupRequires3')
+ assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+ assert 'Running setup.py install for simple\n' in str(res), str(res)
+ assert 'Running setup.py install for simple2\n' in str(res), str(res)
+ assert 'Running setup.py install for SetupRequires3\n' in str(res), str(res)