From 170d972faf225fd6bb094d51e1ab5db086b1e132 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 24 Mar 2015 21:13:07 +1300 Subject: [PATCH] Support declarative requirements. This adds support per the discussion we had on distutils-sig for declarative requirements in setup.cfg. Supported are requires-setup (the problem to be solved). requires-dist (defined by d2to1, and needed as a consequences of supporting setup requirements, because we can't run egg_info during the pre-installation phase to detect requires. Similarly extras are supported, for the same reason. Neither setup-requires nor extras are defined by d2to1, so we may want to bikeshed a little over the syntax there, but the basic structure works. --- docs/reference/pip_install.rst | 78 +++++++++------- pip/req/req_set.py | 85 +++++++++++++++++- tests/data/packages/README.txt | 33 +++++++ .../data/packages/SetupRequires-0.0.1.tar.gz | Bin 0 -> 788 bytes tests/data/packages/SetupRequires/setup.cfg | 4 + tests/data/packages/SetupRequires/setup.py | 8 ++ .../SetupRequires/setuprequires/__init__.py | 0 .../data/packages/SetupRequires2-0.0.1.tar.gz | Bin 0 -> 785 bytes tests/data/packages/SetupRequires2/setup.cfg | 10 +++ tests/data/packages/SetupRequires2/setup.py | 7 ++ .../SetupRequires2/setuprequires2/__init__.py | 0 tests/data/packages/SetupRequires3/setup.cfg | 4 + tests/data/packages/SetupRequires3/setup.py | 7 ++ .../SetupRequires3/setuprequires3/__init__.py | 0 tests/functional/test_install.py | 31 +++++++ 15 files changed, 234 insertions(+), 33 deletions(-) create mode 100644 tests/data/packages/SetupRequires-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires/setup.cfg create mode 100644 tests/data/packages/SetupRequires/setup.py create mode 100644 tests/data/packages/SetupRequires/setuprequires/__init__.py create mode 100644 tests/data/packages/SetupRequires2-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires2/setup.cfg create mode 100644 tests/data/packages/SetupRequires2/setup.py create mode 100644 tests/data/packages/SetupRequires2/setuprequires2/__init__.py create mode 100644 tests/data/packages/SetupRequires3/setup.cfg create mode 100644 tests/data/packages/SetupRequires3/setup.py create mode 100644 tests/data/packages/SetupRequires3/setuprequires3/__init__.py diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 7eb37e6a73f..28d37065819 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -337,45 +337,28 @@ the project path. This is one advantage over just using ``setup.py develop``, which creates the "egg-info" directly relative the current working directory. +Build System Interface +++++++++++++++++++++++ -Controlling setup_requires -++++++++++++++++++++++++++ - -Setuptools offers the ``setup_requires`` `setup() keyword -`_ -for specifying dependencies that need to be present in order for the `setup.py` -script to run. Internally, Setuptools uses ``easy_install`` to fulfill these -dependencies. - -pip has no way to control how these dependencies are located. None of the -Package Index Options have an effect. - -The solution is to configure a "system" or "personal" `Distutils configuration -file -`_ to -manage the fulfillment. - -For example, to have the dependency located at an alternate index, add this: - -:: +In order for pip to install a package from source, pip must recognise the build +system. Today only one build system is recognised, with two variants. - [easy_install] - index_url = https://my.index-mirror.com +If there is a ``setup.cfg`` with any of ``[extras]``, ``requires-dist`` or +``requires-setup`` present then a declarative setuptools package is detected. -To have the dependency located from a local directory and not crawl PyPI, add this: +Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed. -:: +Declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~ - [easy_install] - allow_hosts = '' - find_links = file:///path/to/local/archives +``setup.py`` must implement the following commands:: + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] -Build System Interface -++++++++++++++++++++++ +Non-declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order for pip to install a package from source, ``setup.py`` must implement -the following commands:: +``setup.py`` must implement the following commands:: setup.py egg_info [--egg-base XXX] setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] @@ -409,6 +392,39 @@ Installing a package from a wheel does not invoke the build system at all. .. _PyPI: http://pypi.python.org/pypi/ .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies +Controlling setup_requires() +++++++++++++++++++++++++++++ + +Setuptools offers the ``setup_requires`` `setup() keyword +`_ +for specifying dependencies that need to be present in order for the `setup.py` +script to run. Internally, Setuptools uses ``easy_install`` to fulfill these +dependencies. + +pip has no way to control how these dependencies are located. None of the +Package Index Options have an effect. + +The solution is to configure a "system" or "personal" `Distutils configuration +file +`_ to +manage the fulfillment. + +For example, to have the dependency located at an alternate index, add this: + +:: + + [easy_install] + index_url = https://my.index-mirror.com + +To have the dependency located from a local directory and not crawl PyPI, add this: + +:: + + [easy_install] + allow_hosts = '' + find_links = file:///path/to/local/archives + + .. _`pip install Options`: diff --git a/pip/req/req_set.py b/pip/req/req_set.py index dee9c69908a..2de4a0eb6fb 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -7,6 +7,7 @@ from pip._vendor import pkg_resources from pip._vendor import requests +from pip._vendor.six.moves import configparser from pip.download import (url_to_path, unpack_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, @@ -80,6 +81,15 @@ def prep_for_dist(self): raise NotImplementedError(self.dist) +def _sdist_or_static(req_to_install): + result = IsStaticMetadata(req_to_install) + try: + result.dist(None) + return result + except (configparser.NoSectionError, configparser.NoOptionError, AssertionError): + return IsSDist(req_to_install) + + def make_abstract_dist(req_to_install): """Factory to make an abstract dist object. @@ -89,11 +99,11 @@ def make_abstract_dist(req_to_install): :return: A concrete DistAbstraction. """ if req_to_install.editable: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) elif req_to_install.link and req_to_install.link.is_wheel: return IsWheel(req_to_install) else: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) class IsWheel(DistAbstraction): @@ -123,6 +133,68 @@ def prep_for_dist(self): self.req_to_install.assert_source_matches_version() +class SetupCfgDistribution(pkg_resources.Distribution): + + def __init__(self, cfg, location=None): + self._cfg = cfg + project_name = cfg.get('metadata', 'name') + super(SetupCfgDistribution, self).__init__( + location=location, project_name=project_name) + + @property + def _dep_map(self): + try: + return self.__dep_map + except AttributeError: + # requires + try: + requires = self._cfg.get('metadata', 'requires-dist') + except configparser.NoOptionError: + requires = [] + extras = ( + self._cfg.has_section('extras') + and self._cfg.items('extras')) or [] + dm = {} + dm.setdefault(None, []).extend( + pkg_resources.parse_requirements(requires)) + for extra, extra_reqs in extras: + dm.setdefault(extra, []).extend( + pkg_resources.parse_requirements(extra_reqs)) + return dm + + def setup_requires(self): + try: + setup_requires = self._cfg.get('metadata', 'requires-setup') + except configparser.NoOptionError: + return [] + return pkg_resources.parse_requirements(setup_requires) + + +class IsStaticMetadata(DistAbstraction): + """A static setup.cfg based source tree. + + Must have a name and requires|setup_requires in setup.cfg to be usable. + We look for either requires or setup-requires to handle both the case where + a setup.py has setup-requires but no runtime requirements are declared, and + the case where folk have declaratively expressed their dist requirements + without needing a setup-requires. + """ + def dist(self, finder): + cfg = configparser.SafeConfigParser() + setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg') + cfg.read(setup_cfg) + dist = SetupCfgDistribution(cfg = cfg, location=setup_cfg) + assert dist.project_name + assert dist.requires() or dist.setup_requires() + return dist + + def prep_for_dist(self): + self._dist = self.dist(None) + self.req_to_install.req = pkg_resources.Requirement.parse( + self._dist.project_name) + self.req_to_install._correct_build_location() + + class Installed(DistAbstraction): def dist(self, finder): @@ -518,6 +590,15 @@ def add_req(subreq): self.add_requirement(req_to_install) if not self.ignore_dependencies: + if getattr(dist, 'setup_requires', None): + setup_requires = list(dist.setup_requires()) + if setup_requires: + logger.debug( + "Installing setup_requires: %r", + ','.join(r.project_name for r in setup_requires)) + for subreq in setup_requires: + add_req(subreq) + if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 70f1eb08891..b2f574210e0 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -71,6 +71,39 @@ priority-* ---------- used for testing wheel priority over sdists +SetupRequires +------------- + +has a setup.cfg declaring a setup-requires on upper, and a setup.py that will +fail to import if upper is not installed. + +SetupRequires-0.0.1.tar.gz +-------------------------- + +SetupRequires sdisted, for testing transitive setup_requires. + +SetupRequires2 +-------------- + +has a setup.cfg declaring a dist-requires on setuprequires. Covers both +setup-requires in depended-on packages, and setup.cfg with only requires-dist +expressed. +Also in setup.cfg declares two extras - a and b, a which brings in simple +and b which brings in simple2, for testing extras from setup.cfg. + +SetupRequires2-0.0.1.tar.gz +--------------------------- + +SetupRequires2 sdisted, for testing declarative extras. + +SetupRequires3 +-------------- + +requires SetupRequires2[a,b], as using extras for local paths is currently +broken (issue 1236). Ideally SetupRequires3 would have the extras itself +and no requires-dist (to test declarative extras as sole requirements). + + simple[2]-[123].0.tar.gz ------------------------ contains "simple[2]" package; good for basic testing and version logic. diff --git a/tests/data/packages/SetupRequires-0.0.1.tar.gz b/tests/data/packages/SetupRequires-0.0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..037d49ca3215b40e43e5a80e8722b4c0f81ed5b6 GIT binary patch literal 788 zcmV+v1MB=BiwFo42@zES|72-%bT3n7bail2WpQS_5iX1G7U7+fvJ# z-Pr;~E}77@j!4hBkp9}|Hm%oxq#pkfCO7y$FKzwovQ7Uxjg8Z+i!uz=^I(_XcSsDE>|M;?(5*e(69*!nyO^e~yF`*+g+t@8iCaGe42|Bm^8 za(Ht5?8T%nzsPp;ziBo7Z#tHV{J#wrwceivq^|m8jr)%dUg>YfFAvG)@L2cAlxIAd z^2ERNqA-7xnAk2uSDcc{%+l=D5U8>seqd6c89-6zyH9^13-Hwfnc)qni~*t-4)uG!Q- zbx{Ae!aFY6lu4$)U#NYk8GTB|EapRPB@W1s2dW?;lOm3pxEyLH zz^_bC3;8h>&AHBF7DiXrx1rCI{Myf+^V}CXvx%)eA zfBs8N7x#ZV;{UrJrfoj|DgAG$7%^2TaFzd6Fu2J7f7SoxnQ%+**yMl1u^adQZrKvR z|6AZRUN!({q`cJCMvbaprvbeR3bjlgEU$x4Rb%Ltjo}%QEbz{YFq)R@R8dC%rR|7Ez{29)Nz>8 zF`P~dES2WcFgh}w3aMvX^KP22en>U`BaDBgzbb;V3huV9KQkTXLH~!Mzwd`hsC|Es zo!wi1*R^lZ-_G}l#XQT)^=GbQbqo{w|3&@ptM=XXr{3EC)Uj+c&!Zl*ZRp$ zZ?Z$4kPdjb{@)A^L=g1DWRdRvhX3!@{~2Wz{r|z~KR!4H|8MR8jdI)I{Xg~U{oi#c{NGyt zzv!8$2Jrti{;$qsO+Wv0>i)M}6aH_7@7+-OQbd9)(H$BK9x1V7bc6ihynd{cN4w9= z8GY(3x<6o;|Jnc59bm)$XP#a6KV$HJD|`@|&$;IPkLPY58gyWi5s$^t*xUx>BBR_S zWW0=HF3*O>@o0Z^@M%Qerm^T}JP?e=MGQD!ailiX$iykr|ng{-L8AkIWqqxXdFV;273$;q} zOV0000000000000000000000000 P0AQ1!ijs!908jt`xa^z# literal 0 HcmV?d00001 diff --git a/tests/data/packages/SetupRequires2/setup.cfg b/tests/data/packages/SetupRequires2/setup.cfg new file mode 100644 index 00000000000..123b6ffc36e --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = SetupRequires2 +requires-dist = + SetupRequires + +[extras] +a = + simple +b = + simple2 diff --git a/tests/data/packages/SetupRequires2/setup.py b/tests/data/packages/SetupRequires2/setup.py new file mode 100644 index 00000000000..754287ae15c --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires2', + version='0.0.1', + packages=['setuprequires2'], +) diff --git a/tests/data/packages/SetupRequires2/setuprequires2/__init__.py b/tests/data/packages/SetupRequires2/setuprequires2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires3/setup.cfg b/tests/data/packages/SetupRequires3/setup.cfg new file mode 100644 index 00000000000..04f2cec4311 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires3 +requires-dist = + SetupRequires2[a,b] diff --git a/tests/data/packages/SetupRequires3/setup.py b/tests/data/packages/SetupRequires3/setup.py new file mode 100644 index 00000000000..1fea8e52850 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires3', + version='0.0.1', + packages=['setuprequires3'], +) diff --git a/tests/data/packages/SetupRequires3/setuprequires3/__init__.py b/tests/data/packages/SetupRequires3/setuprequires3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 40545b5c88c..2dcb2ddc5a5 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -769,3 +769,34 @@ def test_install_upgrade_editable_depending_on_other_editable(script): script.pip('install', '--upgrade', '--editable', pkgb_path) result = script.pip('list') assert "pkgb" in result.stdout + + +def _test_setup_requires(script, data, options, name): + to_install = data.packages.join(name) + args = ['install'] + options + [to_install, '-f', data.packages] + res = script.pip(*args, expect_error=False) + assert 'Running setup.py install for upper\n' in str(res) + return res + + +def test_install_declarative_setup_requires_editable(script, data): + res = _test_setup_requires(script, data, ['-e'], 'SetupRequires') + assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_setup_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires2') + assert script.site_packages / 'setuprequires2' in res.files_created, res + + +def test_install_declarative_extras(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires3') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + assert 'Running setup.py install for simple\n' in str(res), str(res) + assert 'Running setup.py install for simple2\n' in str(res), str(res) + assert 'Running setup.py install for SetupRequires3\n' in str(res), str(res)