From abcca31e1339a8c525f122add6a6ed8d5ecd432c Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 24 Mar 2015 21:13:07 +1300 Subject: [PATCH] Support declarative requirements. This adds support per the discussion we had on distutils-sig for declarative requirements in setup.cfg. Supported are setup-requires (the problem to be solved). install-requires, needed as a consequences of supporting setup requirements, because we can't run egg_info during the pre-installation phase to detect requires. Similarly extras are supported, for the same reason. For compatibility with d2to1 we support requires-dist as an alias for install-requires in the setup.cfg metadata section. --- docs/reference/pip_install.rst | 73 ++++++++--- pip/req/req_set.py | 116 +++++++++++++++++- tests/data/packages/README.txt | 33 +++++ .../data/packages/SetupRequires-0.0.1.tar.gz | Bin 0 -> 782 bytes tests/data/packages/SetupRequires/setup.cfg | 4 + tests/data/packages/SetupRequires/setup.py | 8 ++ .../SetupRequires/setuprequires/__init__.py | 0 .../data/packages/SetupRequires2-0.0.1.tar.gz | Bin 0 -> 781 bytes tests/data/packages/SetupRequires2/setup.cfg | 10 ++ tests/data/packages/SetupRequires2/setup.py | 7 ++ .../SetupRequires2/setuprequires2/__init__.py | 0 tests/data/packages/SetupRequires3/setup.cfg | 4 + tests/data/packages/SetupRequires3/setup.py | 7 ++ .../SetupRequires3/setuprequires3/__init__.py | 0 tests/functional/test_install.py | 32 +++++ tests/unit/test_req.py | 42 +++++++ 16 files changed, 317 insertions(+), 19 deletions(-) create mode 100644 tests/data/packages/SetupRequires-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires/setup.cfg create mode 100644 tests/data/packages/SetupRequires/setup.py create mode 100644 tests/data/packages/SetupRequires/setuprequires/__init__.py create mode 100644 tests/data/packages/SetupRequires2-0.0.1.tar.gz create mode 100644 tests/data/packages/SetupRequires2/setup.cfg create mode 100644 tests/data/packages/SetupRequires2/setup.py create mode 100644 tests/data/packages/SetupRequires2/setuprequires2/__init__.py create mode 100644 tests/data/packages/SetupRequires3/setup.cfg create mode 100644 tests/data/packages/SetupRequires3/setup.py create mode 100644 tests/data/packages/SetupRequires3/setuprequires3/__init__.py diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 7eb37e6a73f..d712c95ecbe 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -337,9 +337,58 @@ the project path. This is one advantage over just using ``setup.py develop``, which creates the "egg-info" directly relative the current working directory. +Build System Interface +++++++++++++++++++++++ + +In order for pip to install a package from source, pip must recognise the build +system. Today only one build system is recognised, with two variants. + +If there is a ``setup.cfg`` with any of ``[extras]``, ``install-requires``, +``setup-requires``, or ``requires-dist`` present then a declarative setuptools +package is detected. + +Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed. + +Declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] + +With declarative dependencies, easy_install - the ``setup_requires`` keyword +is never triggered, as pip can take care of installing the requirements before +``setup.py`` is invoked. + +``setup.cfg`` must contain the package name, and one or more of setup requires, +install requires or extra requires:: + + [metadata] + name = Example + setup-requires = + somedep + install-requires = + runtimedep + [extras] + one = + anotherdep + tests = + mytestdep + +For compatibility with ``d2to1`` ``requires-dist`` is accepted as an alias for +``install-requires``, though if both are supplied an error will occur. + +Non-declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py egg_info [--egg-base XXX] + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] -Controlling setup_requires -++++++++++++++++++++++++++ +The ``egg_info`` command should create egg metadata for the package, as +described in the setuptools documentation at +http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags Setuptools offers the ``setup_requires`` `setup() keyword `_ @@ -370,19 +419,8 @@ To have the dependency located from a local directory and not crawl PyPI, add th allow_hosts = '' find_links = file:///path/to/local/archives - -Build System Interface -++++++++++++++++++++++ - -In order for pip to install a package from source, ``setup.py`` must implement -the following commands:: - - setup.py egg_info [--egg-base XXX] - setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] - -The ``egg_info`` command should create egg metadata for the package, as -described in the setuptools documentation at -http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags +Common setuptools behaviour +~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``install`` command should implement the complete process of installing the package to the target directory XXX. @@ -404,13 +442,14 @@ Investigate in more detail when this command is required). No other build system commands are invoked by the ``pip install`` command. +Wheels +~~~~~~ + Installing a package from a wheel does not invoke the build system at all. .. _PyPI: http://pypi.python.org/pypi/ .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies - - .. _`pip install Options`: Options diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 65868eb7bd3..e293a1a8d0e 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -7,6 +7,7 @@ from pip._vendor import pkg_resources from pip._vendor import requests +from pip._vendor.six.moves import configparser from pip.download import (url_to_path, unpack_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, @@ -80,6 +81,16 @@ def prep_for_dist(self): raise NotImplementedError(self.dist) +def _sdist_or_static(req_to_install): + result = IsStaticMetadata(req_to_install) + try: + result.dist(None) + return result + except (configparser.NoSectionError, configparser.NoOptionError, + BadStaticSetupCfg): + return IsSDist(req_to_install) + + def make_abstract_dist(req_to_install): """Factory to make an abstract dist object. @@ -89,11 +100,11 @@ def make_abstract_dist(req_to_install): :return: A concrete DistAbstraction. """ if req_to_install.editable: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) elif req_to_install.link and req_to_install.link.is_wheel: return IsWheel(req_to_install) else: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) class IsWheel(DistAbstraction): @@ -123,6 +134,98 @@ def prep_for_dist(self): self.req_to_install.assert_source_matches_version() +class SetupCfgDistribution(pkg_resources.Distribution): + """A Distribution object that consults setup.cfg.""" + + def __init__(self, cfg, location=None): + self._cfg = cfg + project_name = cfg.get('metadata', 'name') + super(SetupCfgDistribution, self).__init__( + location=location, project_name=project_name) + + @property + def _dep_map(self): + # Distribution._dep_map is not generic - its expressed in terms of the + # requires.txt format from within an egg info directory, and the + # metadata interfae within Distribution looks for files-and-lines. + # To provide metadata from a ConfigParser we could either marshall + # the data to a VFS, or we can reimplement this property which is + # the primary worker to obtain requirements. + try: + return self.__dep_map + except AttributeError: + # requires + requires = self._option('install-requires', 'requires-dist') + extras = ( + self._cfg.has_section('extras') + and self._cfg.items('extras')) or [] + dm = {} + dm.setdefault(None, []).extend( + pkg_resources.parse_requirements(requires)) + for extra, extra_reqs in extras: + dm.setdefault(extra, []).extend( + pkg_resources.parse_requirements(extra_reqs)) + self.__dep_map = dm + return dm + + def setup_requires(self): + try: + setup_requires = self._cfg.get('metadata', 'setup-requires') + except configparser.NoOptionError: + return [] + return list(pkg_resources.parse_requirements(setup_requires)) + + def _option(self, option1, option2): + try: + result = self._cfg.get('metadata', option1) + if self._cfg.has_option('metadata', option2): + raise BadStaticSetupCfg('both %s and %s' % (option1, option2)) + return result + except configparser.NoOptionError: + try: + return self._cfg.get('metadata', option2) + except configparser.NoOptionError: + return [] + + +class BadStaticSetupCfg(Exception): + pass + + +def _validate_static(dist): + has_deps = False + for deps in dist._dep_map.values(): + if deps: + has_deps = True + break + if not (dist.project_name and (has_deps or dist.setup_requires())): + raise BadStaticSetupCfg() + + +class IsStaticMetadata(DistAbstraction): + """A static setup.cfg based source tree. + + Must have a name and requires|setup_requires in setup.cfg to be usable. + We look for either requires or setup-requires to handle both the case where + a setup.py has setup-requires but no runtime requirements are declared, and + the case where folk have declaratively expressed their dist requirements + without needing a setup-requires. + """ + def dist(self, finder): + cfg = configparser.SafeConfigParser() + setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg') + cfg.read(setup_cfg) + dist = SetupCfgDistribution(cfg=cfg, location=setup_cfg) + _validate_static(dist) + return dist + + def prep_for_dist(self): + self._dist = self.dist(None) + self.req_to_install.req = pkg_resources.Requirement.parse( + self._dist.project_name) + self.req_to_install._correct_build_location() + + class Installed(DistAbstraction): def dist(self, finder): @@ -526,6 +629,15 @@ def add_req(subreq): self.add_requirement(req_to_install) if not self.ignore_dependencies: + if getattr(dist, 'setup_requires', None): + setup_requires = dist.setup_requires() + if setup_requires: + logger.debug( + "Installing setup_requires: %r", + ','.join(r.project_name for r in setup_requires)) + for subreq in setup_requires: + add_req(subreq) + if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 70f1eb08891..b2f574210e0 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -71,6 +71,39 @@ priority-* ---------- used for testing wheel priority over sdists +SetupRequires +------------- + +has a setup.cfg declaring a setup-requires on upper, and a setup.py that will +fail to import if upper is not installed. + +SetupRequires-0.0.1.tar.gz +-------------------------- + +SetupRequires sdisted, for testing transitive setup_requires. + +SetupRequires2 +-------------- + +has a setup.cfg declaring a dist-requires on setuprequires. Covers both +setup-requires in depended-on packages, and setup.cfg with only requires-dist +expressed. +Also in setup.cfg declares two extras - a and b, a which brings in simple +and b which brings in simple2, for testing extras from setup.cfg. + +SetupRequires2-0.0.1.tar.gz +--------------------------- + +SetupRequires2 sdisted, for testing declarative extras. + +SetupRequires3 +-------------- + +requires SetupRequires2[a,b], as using extras for local paths is currently +broken (issue 1236). Ideally SetupRequires3 would have the extras itself +and no requires-dist (to test declarative extras as sole requirements). + + simple[2]-[123].0.tar.gz ------------------------ contains "simple[2]" package; good for basic testing and version logic. diff --git a/tests/data/packages/SetupRequires-0.0.1.tar.gz b/tests/data/packages/SetupRequires-0.0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..8f791e8a2a238cab1105eddb3e6e64a9ae8d1e50 GIT binary patch literal 782 zcmV+p1M&PHiwFo$6BAVe|72-%bT3n7bail2WpQ##9-Lz_OF-?)zx*;rII8Mcc`0pfX!`5h}jAfyS_qnlr zoI0wG-*ZlmCqiY(srZulQlvfFr+v1sH4M#h9IF_aW6|4Vg~kRJHEqT$=4g~zre$lo zrL{noDlTyLpzH;3LYZ%1;JH6M4{_|-~ z|28#j=)Vn6eL?@fssFWz+hqQy%&zI*EC%Smt^8jlP6Pw!f3yCpimdVe54D_n{cl?6 z|Fpulj-R?x1cIlcGt}oiNX4Gs@$&L{)ym1O9rM6AU`%wrLoolZ{;Lnb#`QmNjJp1* zjr`vV?}g$su6XamQtd;X^(h(iPz?38b3lGRP#zL8$-!E%&J{%u^9Fw$`q{jWDu9{!H| z_kXG3;Qnt%{QvM~+V=hrUBCYuanHTX{9snB7n6L$Vk=rs(nXYiVJ!Tj zdMJ&^+`Fy+KQ8o}s{iwVkf#CYAN4=#e*gdg00000000000000000000000000Du}l M0E~zb6aY{F0JbofzyJUM literal 0 HcmV?d00001 diff --git a/tests/data/packages/SetupRequires/setup.cfg b/tests/data/packages/SetupRequires/setup.cfg new file mode 100644 index 00000000000..0a7a91e58b5 --- /dev/null +++ b/tests/data/packages/SetupRequires/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires +setup-requires = + upper diff --git a/tests/data/packages/SetupRequires/setup.py b/tests/data/packages/SetupRequires/setup.py new file mode 100644 index 00000000000..a6a6cdecaf7 --- /dev/null +++ b/tests/data/packages/SetupRequires/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup +import upper + +setup( + name='SetupRequires', + version='0.0.1', + packages=['setuprequires'], +) diff --git a/tests/data/packages/SetupRequires/setuprequires/__init__.py b/tests/data/packages/SetupRequires/setuprequires/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires2-0.0.1.tar.gz b/tests/data/packages/SetupRequires2-0.0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..b426544a71b138a2191a8992fdc77123a0c22e4e GIT binary patch literal 781 zcmV+o1M>VIiwFp56BAVe|72-%bT3n7bail2WpQa%{HjjwEulc+O#EGugkJd`+g5t zJ~q1_SSih=VRU3V6H?Ew&AaV*^+T%hA7Sz<{Z$c^RdBm?{h8_5CiH(S`ul#EgxdE9 z+3ADzcU}8-{cYx1PQFeUbM4&!CiMS{`ae|d`|D4=n*P*d`2hV}U@6nsDD~E98Yv@; zvsCJ{2qES0lo&Y)kHx6Bxm^JqdO-51w+u*F# z&zAwInm)Ns{r%lH{deQnd*lwwrEcM-3V{Xebw-(|Uj;QtoDMhyPHv;Q~BZG+eU)T{S@*Jbd3YyJPCXQCRw z|2O!*+K)AT|IexW-*PGZ-wNNlq4K4O1XrRvG?qM4V$0|T`NsMDwoV@H+B?<`rK0;C zhWVfUUtIwk&1IPB@@!RbKjM0s z%J?z@_x>UOU)&UG<{Y>B-*#%x|2@ma^?xfI#peZJN{S~u#;ET4Fv9v2LJ#700000000000000000000000000QiX? LUy@tY08jt`sgIz| literal 0 HcmV?d00001 diff --git a/tests/data/packages/SetupRequires2/setup.cfg b/tests/data/packages/SetupRequires2/setup.cfg new file mode 100644 index 00000000000..25034eabc38 --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = SetupRequires2 +install-requires = + SetupRequires + +[extras] +a = + simple +b = + simple2 diff --git a/tests/data/packages/SetupRequires2/setup.py b/tests/data/packages/SetupRequires2/setup.py new file mode 100644 index 00000000000..754287ae15c --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires2', + version='0.0.1', + packages=['setuprequires2'], +) diff --git a/tests/data/packages/SetupRequires2/setuprequires2/__init__.py b/tests/data/packages/SetupRequires2/setuprequires2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires3/setup.cfg b/tests/data/packages/SetupRequires3/setup.cfg new file mode 100644 index 00000000000..f798a47fd4a --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires3 +install-requires = + SetupRequires2[a,b] diff --git a/tests/data/packages/SetupRequires3/setup.py b/tests/data/packages/SetupRequires3/setup.py new file mode 100644 index 00000000000..1fea8e52850 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires3', + version='0.0.1', + packages=['setuprequires3'], +) diff --git a/tests/data/packages/SetupRequires3/setuprequires3/__init__.py b/tests/data/packages/SetupRequires3/setuprequires3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 42652358968..b2a46788f1b 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -732,3 +732,35 @@ def test_install_upgrade_editable_depending_on_other_editable(script): script.pip('install', '--upgrade', '--editable', pkgb_path) result = script.pip('list') assert "pkgb" in result.stdout + + +def _test_setup_requires(script, data, options, name): + to_install = data.packages.join(name) + args = ['install'] + options + [to_install, '-f', data.packages] + res = script.pip(*args, expect_error=False) + assert 'Running setup.py install for upper\n' in str(res) + return res + + +def test_install_declarative_setup_requires_editable(script, data): + res = _test_setup_requires(script, data, ['-e'], 'SetupRequires') + assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_setup_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires2') + assert script.site_packages / 'setuprequires2' in res.files_created, res + + +def test_install_declarative_extras(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires3') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + assert 'Running setup.py install for simple\n' in str(res), str(res) + assert 'Running setup.py install for simple2\n' in str(res), str(res) + assert 'Running setup.py install for SetupRequires3\n' in str(res), \ + str(res) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 47ecbabc3ac..bceaebd7e3b 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -15,8 +15,10 @@ from pip.req import (InstallRequirement, RequirementSet, Requirements, parse_requirements) from pip.req.req_install import parse_editable, _filter_install +from pip.req import req_set from pip.utils import read_text_file from pip._vendor import pkg_resources +from pip._vendor.six.moves import configparser from tests.lib import assert_raises_regexp @@ -460,3 +462,43 @@ def test_filter_install(): INFO, 'foo bar') assert _filter_install('I made a SyntaxError') == ( INFO, 'I made a SyntaxError') + + +def _base_cfg(): + cfg = configparser.SafeConfigParser() + cfg.add_section('metadata') + cfg.set('metadata', 'name', 'foo') + return cfg + + +def test_setup_cfg_no_deps(): + cfg = _base_cfg() + dist = req_set.SetupCfgDistribution(cfg, None) + with pytest.raises(req_set.BadStaticSetupCfg): + req_set._validate_static(dist) + + +def _setup_cfg_has_option(option): + cfg = _base_cfg() + cfg.set('metadata', option, 'dep') + dist = req_set.SetupCfgDistribution(cfg, None) + req_set._validate_static(dist) + + +def _setup_cfg_conflicting_option(option1, option2): + cfg = _base_cfg() + cfg.set('metadata', option1, 'dep') + cfg.set('metadata', option2, 'dep') + dist = req_set.SetupCfgDistribution(cfg, None) + with pytest.raises(req_set.BadStaticSetupCfg): + req_set._validate_static(dist) + + +def test_setup_cfg_setup_requires(): + _setup_cfg_has_option('setup-requires') + + +def test_setup_cfg_install_requires(): + _setup_cfg_has_option('install-requires') + _setup_cfg_has_option('requires-dist') + _setup_cfg_conflicting_option('install-requires', 'requires-dist')