From e99c9d7e929e20de768b695f555defef2dc56abb Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 24 Mar 2015 21:13:07 +1300 Subject: [PATCH] Support declarative requirements. This adds support per the discussion we had on distutils-sig for declarative requirements in setup.cfg. Supported are requires-setup (the problem to be solved). requires-dist (defined by d2to1, and needed as a consequences of supporting setup requirements, because we can't run egg_info during the pre-installation phase to detect requires. Similarly extras are supported, for the same reason. Neither setup-requires nor extras are defined by d2to1, so we may want to bikeshed a little over the syntax there, but the basic structure works. --- docs/reference/pip_install.rst | 69 +++++++++--- pip/req/req_set.py | 103 +++++++++++++++++- tests/data/packages/README.txt | 33 ++++++ .../data/packages/SetupRequires-0.0.1.tar.gz | Bin 0 -> 788 bytes tests/data/packages/SetupRequires/setup.cfg | 4 + tests/data/packages/SetupRequires/setup.py | 8 ++ .../SetupRequires/setuprequires/__init__.py | 0 .../data/packages/SetupRequires2-0.0.1.tar.gz | Bin 0 -> 785 bytes tests/data/packages/SetupRequires2/setup.cfg | 10 ++ tests/data/packages/SetupRequires2/setup.py | 7 ++ .../SetupRequires2/setuprequires2/__init__.py | 0 tests/data/packages/SetupRequires3/setup.cfg | 4 + tests/data/packages/SetupRequires3/setup.py | 7 ++ .../SetupRequires3/setuprequires3/__init__.py | 0 tests/functional/test_install.py | 32 ++++++ 15 files changed, 258 insertions(+), 19 deletions(-) create mode 100644 tests/data/packages/SetupRequires-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires/setup.cfg create mode 100644 tests/data/packages/SetupRequires/setup.py create mode 100644 tests/data/packages/SetupRequires/setuprequires/__init__.py create mode 100644 tests/data/packages/SetupRequires2-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires2/setup.cfg create mode 100644 tests/data/packages/SetupRequires2/setup.py create mode 100644 tests/data/packages/SetupRequires2/setuprequires2/__init__.py create mode 100644 tests/data/packages/SetupRequires3/setup.cfg create mode 100644 tests/data/packages/SetupRequires3/setup.py create mode 100644 tests/data/packages/SetupRequires3/setuprequires3/__init__.py diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 7eb37e6a73f..a9c360c989e 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -337,9 +337,54 @@ the project path. This is one advantage over just using ``setup.py develop``, which creates the "egg-info" directly relative the current working directory. +Build System Interface +++++++++++++++++++++++ + +In order for pip to install a package from source, pip must recognise the build +system. Today only one build system is recognised, with two variants. + +If there is a ``setup.cfg`` with any of ``[extras]``, ``requires-dist`` or +``requires-setup`` present then a declarative setuptools package is detected. + +Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed. + +Declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] -Controlling setup_requires -++++++++++++++++++++++++++ +With declarative dependencies, easy_install - the ``setup_requires`` keyword +is never triggered, as pip can take care of installing the requirements before +``setup.py`` is invoked. + +``setup.cfg`` must contain the package name, and one or more of setup requires, +install requires or extra requires:: + + [metadata] + name = Example + requires-setup = + somedep + requires-dist = + runtimedep + [extras] + one = + anotherdep + tests = + mytestdep + +Non-declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py egg_info [--egg-base XXX] + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] + +The ``egg_info`` command should create egg metadata for the package, as +described in the setuptools documentation at +http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags Setuptools offers the ``setup_requires`` `setup() keyword `_ @@ -370,19 +415,8 @@ To have the dependency located from a local directory and not crawl PyPI, add th allow_hosts = '' find_links = file:///path/to/local/archives - -Build System Interface -++++++++++++++++++++++ - -In order for pip to install a package from source, ``setup.py`` must implement -the following commands:: - - setup.py egg_info [--egg-base XXX] - setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] - -The ``egg_info`` command should create egg metadata for the package, as -described in the setuptools documentation at -http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags +Common setuptools behaviour +~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``install`` command should implement the complete process of installing the package to the target directory XXX. @@ -404,13 +438,14 @@ Investigate in more detail when this command is required). No other build system commands are invoked by the ``pip install`` command. +Wheels +~~~~~~ + Installing a package from a wheel does not invoke the build system at all. .. _PyPI: http://pypi.python.org/pypi/ .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies - - .. _`pip install Options`: Options diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 65868eb7bd3..a3d07b35cdf 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -7,6 +7,7 @@ from pip._vendor import pkg_resources from pip._vendor import requests +from pip._vendor.six.moves import configparser from pip.download import (url_to_path, unpack_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, @@ -80,6 +81,16 @@ def prep_for_dist(self): raise NotImplementedError(self.dist) +def _sdist_or_static(req_to_install): + result = IsStaticMetadata(req_to_install) + try: + result.dist(None) + return result + except (configparser.NoSectionError, configparser.NoOptionError, + NoSetupCfgDeps): + return IsSDist(req_to_install) + + def make_abstract_dist(req_to_install): """Factory to make an abstract dist object. @@ -89,11 +100,11 @@ def make_abstract_dist(req_to_install): :return: A concrete DistAbstraction. """ if req_to_install.editable: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) elif req_to_install.link and req_to_install.link.is_wheel: return IsWheel(req_to_install) else: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) class IsWheel(DistAbstraction): @@ -123,6 +134,85 @@ def prep_for_dist(self): self.req_to_install.assert_source_matches_version() +class SetupCfgDistribution(pkg_resources.Distribution): + """A Distribution object that consults setup.cfg.""" + + def __init__(self, cfg, location=None): + self._cfg = cfg + project_name = cfg.get('metadata', 'name') + super(SetupCfgDistribution, self).__init__( + location=location, project_name=project_name) + + @property + def _dep_map(self): + # Distribution._dep_map is not generic - its expressed in terms of the + # requires.txt format from within an egg info directory, and the + # metadata interfae within Distribution looks for files-and-lines. + # To provide metadata from a ConfigParser we could either marshall + # the data to a VFS, or we can reimplement this property which is + # the primary worker to obtain requirements. + try: + return self.__dep_map + except AttributeError: + # requires + try: + requires = self._cfg.get('metadata', 'requires-dist') + except configparser.NoOptionError: + requires = [] + extras = ( + self._cfg.has_section('extras') + and self._cfg.items('extras')) or [] + dm = {} + dm.setdefault(None, []).extend( + pkg_resources.parse_requirements(requires)) + for extra, extra_reqs in extras: + dm.setdefault(extra, []).extend( + pkg_resources.parse_requirements(extra_reqs)) + self.__dep_map = dm + return dm + + def setup_requires(self): + try: + setup_requires = self._cfg.get('metadata', 'requires-setup') + except configparser.NoOptionError: + return [] + return pkg_resources.parse_requirements(setup_requires) + + +class NoSetupCfgDeps(Exception): + pass + + +class IsStaticMetadata(DistAbstraction): + """A static setup.cfg based source tree. + + Must have a name and requires|setup_requires in setup.cfg to be usable. + We look for either requires or setup-requires to handle both the case where + a setup.py has setup-requires but no runtime requirements are declared, and + the case where folk have declaratively expressed their dist requirements + without needing a setup-requires. + """ + def dist(self, finder): + cfg = configparser.SafeConfigParser() + setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg') + cfg.read(setup_cfg) + dist = SetupCfgDistribution(cfg=cfg, location=setup_cfg) + has_deps = False + for deps in dist._dep_map.values(): + if deps: + has_deps = True + break + if not (dist.project_name or has_deps or dist.setup_requires()): + raise NoSetupCfgDeps() + return dist + + def prep_for_dist(self): + self._dist = self.dist(None) + self.req_to_install.req = pkg_resources.Requirement.parse( + self._dist.project_name) + self.req_to_install._correct_build_location() + + class Installed(DistAbstraction): def dist(self, finder): @@ -526,6 +616,15 @@ def add_req(subreq): self.add_requirement(req_to_install) if not self.ignore_dependencies: + if getattr(dist, 'setup_requires', None): + setup_requires = list(dist.setup_requires()) + if setup_requires: + logger.debug( + "Installing setup_requires: %r", + ','.join(r.project_name for r in setup_requires)) + for subreq in setup_requires: + add_req(subreq) + if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 70f1eb08891..b2f574210e0 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -71,6 +71,39 @@ priority-* ---------- used for testing wheel priority over sdists +SetupRequires +------------- + +has a setup.cfg declaring a setup-requires on upper, and a setup.py that will +fail to import if upper is not installed. + +SetupRequires-0.0.1.tar.gz +-------------------------- + +SetupRequires sdisted, for testing transitive setup_requires. + +SetupRequires2 +-------------- + +has a setup.cfg declaring a dist-requires on setuprequires. Covers both +setup-requires in depended-on packages, and setup.cfg with only requires-dist +expressed. +Also in setup.cfg declares two extras - a and b, a which brings in simple +and b which brings in simple2, for testing extras from setup.cfg. + +SetupRequires2-0.0.1.tar.gz +--------------------------- + +SetupRequires2 sdisted, for testing declarative extras. + +SetupRequires3 +-------------- + +requires SetupRequires2[a,b], as using extras for local paths is currently +broken (issue 1236). Ideally SetupRequires3 would have the extras itself +and no requires-dist (to test declarative extras as sole requirements). + + simple[2]-[123].0.tar.gz ------------------------ contains "simple[2]" package; good for basic testing and version logic. diff --git a/tests/data/packages/SetupRequires-0.0.1.tar.gz b/tests/data/packages/SetupRequires-0.0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..037d49ca3215b40e43e5a80e8722b4c0f81ed5b6 GIT binary patch literal 788 zcmV+v1MB=BiwFo42@zES|72-%bT3n7bail2WpQS_5iX1G7U7+fvJ# z-Pr;~E}77@j!4hBkp9}|Hm%oxq#pkfCO7y$FKzwovQ7Uxjg8Z+i!uz=^I(_XcSsDE>|M;?(5*e(69*!nyO^e~yF`*+g+t@8iCaGe42|Bm^8 za(Ht5?8T%nzsPp;ziBo7Z#tHV{J#wrwceivq^|m8jr)%dUg>YfFAvG)@L2cAlxIAd z^2ERNqA-7xnAk2uSDcc{%+l=D5U8>seqd6c89-6zyH9^13-Hwfnc)qni~*t-4)uG!Q- zbx{Ae!aFY6lu4$)U#NYk8GTB|EapRPB@W1s2dW?;lOm3pxEyLH zz^_bC3;8h>&AHBF7DiXrx1rCI{Myf+^V}CXvx%)eA zfBs8N7x#ZV;{UrJrfoj|DgAG$7%^2TaFzd6Fu2J7f7SoxnQ%+**yMl1u^adQZrKvR z|6AZRUN!({q`cJCMvbaprvbeR3bjlgEU$x4Rb%Ltjo}%QEbz{YFq)R@R8dC%rR|7Ez{29)Nz>8 zF`P~dES2WcFgh}w3aMvX^KP22en>U`BaDBgzbb;V3huV9KQkTXLH~!Mzwd`hsC|Es zo!wi1*R^lZ-_G}l#XQT)^=GbQbqo{w|3&@ptM=XXr{3EC)Uj+c&!Zl*ZRp$ zZ?Z$4kPdjb{@)A^L=g1DWRdRvhX3!@{~2Wz{r|z~KR!4H|8MR8jdI)I{Xg~U{oi#c{NGyt zzv!8$2Jrti{;$qsO+Wv0>i)M}6aH_7@7+-OQbd9)(H$BK9x1V7bc6ihynd{cN4w9= z8GY(3x<6o;|Jnc59bm)$XP#a6KV$HJD|`@|&$;IPkLPY58gyWi5s$^t*xUx>BBR_S zWW0=HF3*O>@o0Z^@M%Qerm^T}JP?e=MGQD!ailiX$iykr|ng{-L8AkIWqqxXdFV;273$;q} zOV0000000000000000000000000 P0AQ1!ijs!908jt`xa^z# literal 0 HcmV?d00001 diff --git a/tests/data/packages/SetupRequires2/setup.cfg b/tests/data/packages/SetupRequires2/setup.cfg new file mode 100644 index 00000000000..123b6ffc36e --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = SetupRequires2 +requires-dist = + SetupRequires + +[extras] +a = + simple +b = + simple2 diff --git a/tests/data/packages/SetupRequires2/setup.py b/tests/data/packages/SetupRequires2/setup.py new file mode 100644 index 00000000000..754287ae15c --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires2', + version='0.0.1', + packages=['setuprequires2'], +) diff --git a/tests/data/packages/SetupRequires2/setuprequires2/__init__.py b/tests/data/packages/SetupRequires2/setuprequires2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires3/setup.cfg b/tests/data/packages/SetupRequires3/setup.cfg new file mode 100644 index 00000000000..04f2cec4311 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires3 +requires-dist = + SetupRequires2[a,b] diff --git a/tests/data/packages/SetupRequires3/setup.py b/tests/data/packages/SetupRequires3/setup.py new file mode 100644 index 00000000000..1fea8e52850 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires3', + version='0.0.1', + packages=['setuprequires3'], +) diff --git a/tests/data/packages/SetupRequires3/setuprequires3/__init__.py b/tests/data/packages/SetupRequires3/setuprequires3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 42652358968..b2a46788f1b 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -732,3 +732,35 @@ def test_install_upgrade_editable_depending_on_other_editable(script): script.pip('install', '--upgrade', '--editable', pkgb_path) result = script.pip('list') assert "pkgb" in result.stdout + + +def _test_setup_requires(script, data, options, name): + to_install = data.packages.join(name) + args = ['install'] + options + [to_install, '-f', data.packages] + res = script.pip(*args, expect_error=False) + assert 'Running setup.py install for upper\n' in str(res) + return res + + +def test_install_declarative_setup_requires_editable(script, data): + res = _test_setup_requires(script, data, ['-e'], 'SetupRequires') + assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_setup_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires2') + assert script.site_packages / 'setuprequires2' in res.files_created, res + + +def test_install_declarative_extras(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires3') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + assert 'Running setup.py install for simple\n' in str(res), str(res) + assert 'Running setup.py install for simple2\n' in str(res), str(res) + assert 'Running setup.py install for SetupRequires3\n' in str(res), \ + str(res)