Allow PEP508 url dependencies in install_requires

news/4187.feature

@@ -0,0 +1,5 @@
+Allow PEP 508 URL requirements to be used as dependencies.
+
+As a security measure, pip will raise an exception when installing packages from
+PyPI if those packages depend on packages not also hosted on PyPI.
+In the future, PyPI will block uploading packages with such external URL dependencies directly.

src/pip/_internal/models/index.py

@@ -2,14 +2,20 @@
 
 
 class Index(object):
- def __init__(self, url):
+ def __init__(self, url, file_storage_domain):
  self.url = url
  self.netloc = urllib_parse.urlsplit(url).netloc
  self.simple_url = self.url_to_path('simple')
  self.pypi_url = self.url_to_path('pypi')
 
+ # This is part of a temporary hack used to block installs of PyPI
+ # packages which depend on external urls only necessary until PyPI can
+ # block such packages themselves
+ self.file_storage_domain = file_storage_domain
+
  def url_to_path(self, path):
  return urllib_parse.urljoin(self.url, path)
 
 
-PyPI = Index('https://pypi.org/')
+PyPI = Index('https://pypi.org/', 'files.pythonhosted.org')
+TestPyPI = Index('https://test.pypi.org/', 'test-files.pythonhosted.org')

src/pip/_internal/req/req_install.py

@@ -32,6 +32,7 @@
 from pip._internal.locations import (
  PIP_DELETE_MARKER_FILENAME, running_under_virtualenv,
 )
+from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.req.req_uninstall import UninstallPathSet
 from pip._internal.utils.deprecation import (
  RemovedInPip11Warning, RemovedInPip12Warning,
@@ -169,11 +170,19 @@ def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None):
  req = Requirement(req)
  except InvalidRequirement:
  raise InstallationError("Invalid requirement: '%s'" % req)
- if req.url:
+
+ domains_not_allowed = [
+ PyPI.file_storage_domain,
+ TestPyPI.file_storage_domain,
+ ]
+ if req.url and comes_from.link.netloc in domains_not_allowed:
+ # Explicitly disallow pypi packages that depend on external urls
  raise InstallationError(
- "Direct url requirement (like %s) are not allowed for "
- "dependencies" % req
+ "Packages installed from PyPI cannot depend on packages "
+ "which are not also hosted on PyPI.\n"
+ "%s depends on %s " % (comes_from.name, req)
  )
+
  return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache)
 
  @classmethod

tests/functional/test_install.py

@@ -8,6 +8,7 @@
 import pytest
 
 from pip._internal import pep425tags
+from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.status_codes import ERROR
 from pip._internal.utils.misc import rmtree
 from tests.lib import (
@@ -1267,9 +1268,31 @@ def test_install_pep508_with_url_in_install_requires(script):
  'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4'
  ],
  )
- res = script.pip('install', pkga_path, expect_error=True)
- assert "Direct url requirement " in res.stderr, str(res)
- assert "are not allowed for dependencies" in res.stderr, str(res)
+ res = script.pip('install', pkga_path)
+ assert "Successfully installed packaging-15.3" in str(res), str(res)
+
+
+@pytest.mark.network
+@pytest.mark.parametrize('index', (PyPI.simple_url, TestPyPI.simple_url))
+def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index):
+ res = script.pip(
+ 'install',
+ '--index-url',
+ index,
+ 'pep-508-url-deps',
+ expect_error=True,
+ )
+ error_message = (
+ "Packages installed from PyPI cannot depend on packages "
+ "which are not also hosted on PyPI."
+ )
+ error_cause = (
+ "pep-508-url-deps depends on sampleproject@ "
+ "https:/pypa/sampleproject/archive/master.zip"
+ )
+ assert res.returncode == 1
+ assert error_message in res.stderr, str(res)
+ assert error_cause in res.stderr, str(res)
 
 
 def test_installing_scripts_outside_path_prints_warning(script):