Feature/better resolver

pipenv/_compat.py

@@ -29,6 +29,11 @@ def _infer_return_type(*args):
  _types.add(type(arg))
  return _types.pop()
 
+try:
+ from pathlib import Path
+except ImportError:
+ from pathlib2 import Path
+
 
 try:
  from weakref import finalize
@@ -51,8 +56,6 @@ def detach(self):
  class ResourceWarning(Warning):
  pass
 
-# -*- coding=utf-8 -*-
-
 
 def pip_import(module_path, subimport=None, old_path=None):
  internal = 'pip._internal.{0}'.format(module_path)

pipenv/core.py

@@ -46,10 +46,12 @@
  rmtree,
  split_argument,
  extract_uri_from_vcs_dep,
+ fs_str,
 )
 from ._compat import (
  TemporaryDirectory,
- vcs
+ vcs,
+ Path
 )
 from .import pep508checker, progress
 from .environments import (
@@ -73,6 +75,7 @@
  PIPENV_SHELL,
  PIPENV_PYTHON,
  PIPENV_VIRTUALENV,
+ PIPENV_CACHE_DIR,
 )
 
 # Backport required for earlier versions of Python.
@@ -1481,7 +1484,7 @@ def pip_install(
  pre = '--pre' if pre else ''
  quoted_pip = which_pip(allow_global=allow_global)
  quoted_pip = escape_grouped_arguments(quoted_pip)
- upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
+ upgrade_strategy = '--upgrade --upgrade-strategy=to-satisfy-only' if selective_upgrade else ''
  pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
  quoted_pip,
  install_reqs,
@@ -1494,19 +1497,31 @@ def pip_install(
  )
  if verbose:
  click.echo('$ {0}'.format(pip_command), err=True)
- c = delegator.run(pip_command, block=block)
+ cache_dir = Path(PIPENV_CACHE_DIR)
+ pip_config = {
+ 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()),
+ 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()),
+ 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()),
+ }
+ c = delegator.run(pip_command, block=block, env=pip_config)
  return c
 
 
 def pip_download(package_name):
+ cache_dir = Path(PIPENV_CACHE_DIR) 
+ pip_config = {
+ 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()),
+ 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()),
+ 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()),
+ }
  for source in project.sources:
  cmd = '{0} download "{1}" -i {2} -d {3}'.format(
  escape_grouped_arguments(which_pip()),
  package_name,
  source['url'],
  project.download_location,
  )
- c = delegator.run(cmd)
+ c = delegator.run(cmd, env=pip_config)
  if c.return_code == 0:
  break
 

pipenv/patched/piptools/repositories/pypi.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import os
+import sys
 from contextlib import contextmanager
 from shutil import rmtree
 
@@ -20,14 +21,17 @@
  SafeFileCache,
 )
 
-from notpip._vendor.packaging.requirements import InvalidRequirement
+from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from notpip._vendor.packaging.specifiers import SpecifierSet
 from notpip._vendor.pyparsing import ParseException
 
 from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
- make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+ make_install_requirement, format_requirement, dedup)
+
 from .base import BaseRepository
 
 
@@ -159,7 +163,13 @@ def find_best_match(self, ireq, prereleases=None):
  if ireq.editable:
  return ireq # return itself as the best match
 
- all_candidates = self.find_all_candidates(ireq.name)
+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
+ all_candidates = []
+ for c in self.find_all_candidates(ireq.name):
+ if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version):
+ continue
+ all_candidates.append(c)
+
  candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
  try:
  matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
@@ -188,21 +198,33 @@ def get_json_dependencies(self, ireq):
  raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq))
 
  def gen(ireq):
- if self.DEFAULT_INDEX_URL in self.finder.index_urls:
-
- url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
- r = self.session.get(url)
-
- # TODO: Latest isn't always latest.
- latest = list(r.json()['releases'].keys())[-1]
- if str(ireq.req.specifier) == '=={0}'.format(latest):
- latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
- latest_requires = self.session.get(latest_url)
- for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
- i = InstallRequirement.from_line(requires)
+ if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
+ return
+
+ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
+ releases = self.session.get(url).json()['releases']
+
+ matches = [
+ r for r in releases
+ if '=={0}'.format(r) == str(ireq.req.specifier)
+ ]
+ if not matches:
+ return
+
+ release_requires = self.session.get(
+ 'https://pypi.org/pypi/{0}/{1}/json'.format(
+ ireq.req.name, matches[0],
+ ),
+ ).json()
+ try:
+ requires_dist = release_requires['info']['requires_dist']
+ except KeyError:
+ return
 
- if 'extra' not in repr(i.markers):
- yield i
+ for requires in requires_dist:
+ i = InstallRequirement.from_line(requires)
+ if 'extra' not in repr(i.markers):
+ yield i
 
  try:
  if ireq not in self._json_dep_cache:
@@ -226,7 +248,6 @@ def get_dependencies(self, ireq):
 
  return json_results
 
-
  def get_legacy_dependencies(self, ireq):
  """
  Given a pinned or an editable InstallRequirement, returns a set of
@@ -245,7 +266,13 @@ def get_legacy_dependencies(self, ireq):
  setup_requires = self.finder.get_extras_links(
  dist.get_metadata_lines('requires.txt')
  )
- except TypeError:
+ # HACK: Sometimes the InstallRequirement doesn't properly get
+ # these values set on it during the resolution process. It's
+ # difficult to pin down what is going wrong. This fixes things.
+ ireq.version = dist.version
+ ireq.project_name = dist.project_name
+ ireq.req = dist.as_requirement()
+ except (TypeError, ValueError):
  pass
 
  if ireq not in self._dependencies_cache:

pipenv/project.py

@@ -20,7 +20,7 @@
  from pathlib2 import Path
 
 from .cmdparse import Script
-from .vendor.requirementslib import Requirement
+from .vendor.requirementslib.requirements import Requirement
 from .utils import (
  atomic_open_for_write,
  mkdir_p,
@@ -728,8 +728,8 @@ def add_package_to_pipfile(self, package_name, dev=False):
  # Read and append Pipfile.
  p = self.parsed_pipfile
  # Don't re-capitalize file URLs or VCSs.
- package = Requirement.from_line(package_name)
- converted = first(package.as_pipfile().values())
+ package = Requirement.from_line(package_name.strip())
+ _, converted = package.pipfile_entry
  key = 'dev-packages' if dev else 'packages'
  # Set empty group if it doesn't exist yet.
  if key not in p: