Rebase #4501 to resolve merge conflicts

news/4501.feature

@@ -0,0 +1,3 @@
+Installing from a local directory or a VCS URL now builds a wheel to install,
+rather than running ``setup.py install``. Wheels from these sources are not
+cached.

pip/cache.py

@@ -10,6 +10,7 @@
 import pip.index
 from pip.compat import expanduser
 from pip.download import path_to_url
+from pip.utils import temp_dir
 from pip.utils.cache import get_cache_path_for_link
 from pip.wheel import InvalidWheelFilename, Wheel
 
@@ -27,45 +28,62 @@ def __init__(self, cache_dir, format_control):
  binaries being read from the cache.
  """
  self._cache_dir = expanduser(cache_dir) if cache_dir else None
+ # Ephemeral cache: store wheels just for this run
+ self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache")
+ self._ephem_cache_dir.create()
  self._format_control = format_control
 
  def cached_wheel(self, link, package_name):
- not_cached = (
- not self._cache_dir or
- not link or
- link.is_wheel or
- not link.is_artifact or
- not package_name
- )
+ orig_link = link
+ link = cached_wheel(
+ self._cache_dir, link, self._format_control, package_name)
+ if link is orig_link:
+ link = cached_wheel(
+ self._ephem_cache_dir.path, link, self._format_control,
+ package_name)
+ return link
 
- if not_cached:
- return link
+ def cleanup(self):
+ self._ephem_cache_dir.cleanup()
+
+
+def cached_wheel(cache_dir, link, format_control, package_name):
+ not_cached = (
+ not cache_dir or
+ not link or
+ link.is_wheel or
+ not link.is_artifact or
+ not package_name
+ )
 
- canonical_name = canonicalize_name(package_name)
- formats = pip.index.fmt_ctl_formats(
- self._format_control, canonical_name
- )
- if "binary" not in formats:
+ if not_cached:
+ return link
+
+ canonical_name = canonicalize_name(package_name)
+ formats = pip.index.fmt_ctl_formats(
+ format_control, canonical_name
+ )
+ if "binary" not in formats:
+ return link
+ root = get_cache_path_for_link(cache_dir, link)
+ try:
+ wheel_names = os.listdir(root)
+ except OSError as err:
+ if err.errno in {errno.ENOENT, errno.ENOTDIR}:
  return link
- root = get_cache_path_for_link(self._cache_dir, link)
+ raise
+ candidates = []
+ for wheel_name in wheel_names:
  try:
- wheel_names = os.listdir(root)
- except OSError as err:
- if err.errno in {errno.ENOENT, errno.ENOTDIR}:
- return link
- raise
- candidates = []
- for wheel_name in wheel_names:
- try:
- wheel = Wheel(wheel_name)
- except InvalidWheelFilename:
- continue
- if not wheel.supported():
- # Built for a different python/arch/etc
- continue
- candidates.append((wheel.support_index_min(), wheel_name))
- if not candidates:
- return link
- candidates.sort()
- path = os.path.join(root, candidates[0][1])
- return pip.index.Link(path_to_url(path))
+ wheel = Wheel(wheel_name)
+ except InvalidWheelFilename:
+ continue
+ if not wheel.supported():
+ # Built for a different python/arch/etc
+ continue
+ candidates.append((wheel.support_index_min(), wheel_name))
+ if not candidates:
+ return link
+ candidates.sort()
+ path = os.path.join(root, candidates[0][1])
+ return pip.index.Link(path_to_url(path))

pip/commands/freeze.py

@@ -88,5 +88,8 @@ def run(self, options, args):
  skip=skip,
  exclude_editable=options.exclude_editable)
 
- for line in freeze(**freeze_kwargs):
- sys.stdout.write(line + '\n')
+ try:
+ for line in freeze(**freeze_kwargs):
+ sys.stdout.write(line + '\n')
+ finally:
+ wheel_cache.cleanup()

pip/commands/install.py

@@ -247,18 +247,19 @@ def run(self, options, args):
  use_user_site=options.use_user_site,
  )
 
- self.populate_requirement_set(
- requirement_set, args, options, finder, session, self.name,
- wheel_cache
- )
- preparer = RequirementPreparer(
- build_dir=directory.path,
- src_dir=options.src_dir,
- download_dir=None,
- wheel_download_dir=None,
- progress_bar=options.progress_bar,
- )
  try:
+ self.populate_requirement_set(
+ requirement_set, args, options, finder, session,
+ self.name, wheel_cache
+ )
+ preparer = RequirementPreparer(
+ build_dir=directory.path,
+ src_dir=options.src_dir,
+ download_dir=None,
+ wheel_download_dir=None,
+ progress_bar=options.progress_bar,
+ )
+
  resolver = Resolver(
  preparer=preparer,
  finder=finder,
@@ -350,6 +351,7 @@ def run(self, options, args):
  # Clean up
  if not options.no_clean:
  requirement_set.cleanup_files()
+ wheel_cache.cleanup()
 
  if options.target_dir:
  self._handle_target_dir(

pip/commands/wheel.py

@@ -146,35 +146,35 @@ def run(self, options, args):
  require_hashes=options.require_hashes,
  )
 
- self.populate_requirement_set(
- requirement_set, args, options, finder, session, self.name,
- wheel_cache
- )
+ try:
+ self.populate_requirement_set(
+ requirement_set, args, options, finder, session,
+ self.name, wheel_cache
+ )
 
- preparer = RequirementPreparer(
- build_dir=directory.path,
- src_dir=options.src_dir,
- download_dir=None,
- wheel_download_dir=options.wheel_dir,
- progress_bar=options.progress_bar,
- )
+  preparer = RequirementPreparer(
+  build_dir=directory.path,
+  src_dir=options.src_dir,
+  download_dir=None,
+  wheel_download_dir=options.wheel_dir,
+  progress_bar=options.progress_bar,
+  )
 
- resolver = Resolver(
- preparer=preparer,
- finder=finder,
- session=session,
- wheel_cache=wheel_cache,
- use_user_site=False,
- upgrade_strategy="to-satisfy-only",
- force_reinstall=False,
- ignore_dependencies=options.ignore_dependencies,
- ignore_requires_python=options.ignore_requires_python,
- ignore_installed=True,
- isolated=options.isolated_mode,
- )
- resolver.resolve(requirement_set)
+  resolver = Resolver(
+  preparer=preparer,
+  finder=finder,
+  session=session,
+  wheel_cache=wheel_cache,
+  use_user_site=False,
+  upgrade_strategy="to-satisfy-only",
+  force_reinstall=False,
+  ignore_dependencies=options.ignore_dependencies,
+  ignore_requires_python=options.ignore_requires_python,
+  ignore_installed=True,
+  isolated=options.isolated_mode,
+  )
+  resolver.resolve(requirement_set)
 
- try:
  # build wheels
  wb = WheelBuilder(
  requirement_set,
@@ -196,3 +196,4 @@ def run(self, options, args):
  finally:
  if not options.no_clean:
  requirement_set.cleanup_files()
+ wheel_cache.cleanup()

pip/req/req_set.py

@@ -44,9 +44,6 @@ def __init__(self,
  require_hashes=False, target_dir=None, use_user_site=False,
  pycompile=True):
  """Create a RequirementSet.
-
- :param wheel_cache: The pip wheel cache, for passing to
- InstallRequirement.
  """
 
  self.requirements = Requirements()

pip/wheel.py

@@ -736,6 +736,7 @@ def build(self, session, autobuilding=False):
 
  buildset = []
  for req in reqset:
+ ephem_cache = False
  if req.constraint:
  continue
  if req.is_wheel:
@@ -745,44 +746,46 @@ def build(self, session, autobuilding=False):
  elif autobuilding and req.editable:
  pass
  elif autobuilding and req.link and not req.link.is_artifact:
- pass
+ # VCS checkout. Build wheel just for this run.
+ ephem_cache = True
  elif autobuilding and not req.source_dir:
  pass
  else:
  if autobuilding:
  link = req.link
  base, ext = link.splitext()
  if pip.index.egg_info_matches(base, None, link) is None:
- # Doesn't look like a package - don't autobuild a wheel
- # because we'll have no way to lookup the result sanely
- continue
- if "binary" not in pip.index.fmt_ctl_formats(
+ # E.g. local directory. Build wheel just for this run.
+ ephem_cache = True
+ elif "binary" not in pip.index.fmt_ctl_formats(
  self.finder.format_control,
  canonicalize_name(req.name)):
  logger.info(
  "Skipping bdist_wheel for %s, due to binaries "
  "being disabled for it.", req.name)
  continue
- buildset.append(req)
+ buildset.append((req, ephem_cache))
 
  if not buildset:
  return True
 
  # Build the wheels.
  logger.info(
  'Building wheels for collected packages: %s',
- ', '.join([req.name for req in buildset]),
+ ', '.join([req.name for (req, _) in buildset]),
  )
  with indent_log():
  build_success, build_failure = [], []
- for req in buildset:
+ for req, ephem in buildset:
  python_tag = None
  if autobuilding:
  python_tag = pep425tags.implementation_tag
  # NOTE: Should move out a method on the cache directly.
- output_dir = get_cache_path_for_link(
- self.wheel_cache._cache_dir, req.link
+ cache_root = (
+ self.wheel_cache._ephem_cache_dir.path if ephem
+ else self.wheel_cache._cache_dir
  )
+ output_dir = get_cache_path_for_link(cache_root, req.link)
  try:
  ensure_dir(output_dir)
  except OSError as e:

tests/functional/test_install.py

@@ -983,16 +983,15 @@ def test_install_builds_wheels(script, data):
  # and built wheels for upper and wheelbroken
  assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
  assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
- # But not requires_wheel... which is a local dir and thus uncachable.
- assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+ # Wheels are built for local directories, but not cached.
+ assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
  # wheelbroken has to run install
  # into the cache
  assert wheels != [], str(res)
  # and installed from the wheel
  assert "Running setup.py install for upper" not in str(res), str(res)
- # the local tree can't build a wheel (because we can't assume that every
- # build will have a suitable unique key to cache on).
- assert "Running setup.py install for requires-wheel" in str(res), str(res)
+ # Wheels are built for local directories, but not cached.
+ assert "Running setup.py install for requir" not in str(res), str(res)
  # wheelbroken has to run install
  assert "Running setup.py install for wheelb" in str(res), str(res)
  # We want to make sure we used the correct implementation tag
@@ -1019,16 +1018,15 @@ def test_install_no_binary_disables_building_wheels(script, data):
  wheels.extend(files)
  # and built wheels for wheelbroken only
  assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
- # But not requires_wheel... which is a local dir and thus uncachable.
- assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+ # Wheels are built for local directories, but not cached
+ assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
  # Nor upper, which was blacklisted
  assert "Running setup.py bdist_wheel for upper" not in str(res), str(res)
  # wheelbroken has to run install
  # into the cache
  assert wheels != [], str(res)
- # the local tree can't build a wheel (because we can't assume that every
- # build will have a suitable unique key to cache on).
- assert "Running setup.py install for requires-wheel" in str(res), str(res)
+ # Wheels are built for local directories, but not cached
+ assert "Running setup.py install for requir" not in str(res), str(res)
  # And these two fell back to sdist based installed.
  assert "Running setup.py install for wheelb" in str(res), str(res)
  assert "Running setup.py install for upper" in str(res), str(res)

tests/functional/test_install_cleanup.py

@@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
  build = script.base_path / 'pip-build'
  script.pip(
  'install', '--no-clean', '--no-index', '--build', build,
- '--find-links=%s' % data.find_links, 'simple',
+ '--find-links=%s' % data.find_links, 'simple', expect_temp=True,
  )
  assert exists(build)
 
@@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
  result = script.pip(
  'install', '-f', data.find_links, '--no-index', 'simple',
  '--build', build,
- expect_error=True,
+ expect_error=True, expect_temp=True,
  )
 
  assert result.returncode == PREVIOUS_BUILD_DIR_ERROR

tests/functional/test_install_user.py

@@ -68,24 +68,26 @@ def test_install_subversion_usersite_editable_with_distribute(
  )
  result.assert_installed('INITools', use_user_site=True)
 
+ @pytest.mark.network
  def test_install_curdir_usersite(self, script, virtualenv, data):
  """
  Test installing current directory ('.') into usersite
  """
  virtualenv.system_site_packages = True
+ script.pip("install", "wheel")
  run_from = data.packages.join("FSPkg")
  result = script.pip(
  'install', '-vvv', '--user', curdir,
  cwd=run_from,
  expect_error=False,
  )
  fspkg_folder = script.user_site / 'fspkg'
- egg_info_folder = (
- script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion
+ dist_info_folder = (
+ script.user_site / 'FSPkg-0.1.dev0.dist-info'
  )
  assert fspkg_folder in result.files_created, result.stdout
 
- assert egg_info_folder in result.files_created
+ assert dist_info_folder in result.files_created
 
  def test_install_user_venv_nositepkgs_fails(self, script, data):
  """